From f17a60636a074b889105d7e123caf0fa475bebfc Mon Sep 17 00:00:00 2001 From: Andri Date: Mon, 27 Jul 2026 15:39:17 +0000 Subject: [PATCH 1/2] Complete stable protocol transition plans Record ADR-0028 transition evidence. --- crates/kult-node/src/atomic_tests.rs | 1543 ++++++++++++++-- crates/kult-node/src/attachment.rs | 1124 ++++++++---- crates/kult-node/src/authority.rs | 575 ++++-- crates/kult-node/src/devices.rs | 2 +- crates/kult-node/src/groups.rs | 2117 ++++++++++++++++------ crates/kult-node/src/lib.rs | 312 +++- crates/kult-node/src/polls.rs | 13 +- crates/kult-node/tests/groups_e2e.rs | 76 + crates/kult-store/src/backup.rs | 92 +- crates/kult-store/src/commit.rs | 1432 ++++++++++++++- crates/kult-store/src/lib.rs | 29 +- crates/kult-store/src/media.rs | 181 +- docs/00-start-here.md | 1 + docs/03-architecture.md | 9 +- docs/29-stabilization-program.md | 2 +- docs/31-release-evidence-ledger.md | 15 +- docs/34-atomic-transition-inventory.md | 181 ++ docs/adr/0028-atomic-protocol-commits.md | 78 +- docs/adr/README.md | 2 +- 19 files changed, 6237 insertions(+), 1547 deletions(-) create mode 100644 docs/34-atomic-transition-inventory.md diff --git a/crates/kult-node/src/atomic_tests.rs b/crates/kult-node/src/atomic_tests.rs index d77be6d..9ae0c4b 100644 --- a/crates/kult-node/src/atomic_tests.rs +++ b/crates/kult-node/src/atomic_tests.rs @@ -1,10 +1,24 @@ use std::path::PathBuf; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use async_trait::async_trait; use rand::{rngs::StdRng, SeedableRng}; use kult_crypto::KdfProfile; use kult_protocol::{Envelope, EnvelopeKind}; -use kult_store::{CommitFailpoint, CommitFailure, DeliveryState}; +use kult_store::{ + AttachmentStagePlan, AttachmentStatePlan, CommitFailpoint, CommitFailure, DeliveryState, + Direction, GroupDelivery, GroupMember, GroupMessageRecord, GroupRecord, GroupSendPlan, + GroupStatePlan, GroupStateTransition, GroupTransition, MediaDirection, MediaObjectRecord, + MediaRecord, MediaScope, MediaTransferRecord, MediaTransferState, MediaTransferTransition, + MessageDeviceDeliveryRecord, MessageRecord, QueueClass, QueueItem, +}; +use kult_transport::{ + CostClass, DeliveryHint, LatencyClass, LinkProfile, Reachability, SendReceipt, Transport, +}; use super::*; @@ -15,8 +29,46 @@ const TEST_KDF: KdfProfile = KdfProfile { p_cost: 1, }; +#[derive(Default)] +struct CountingTransport { + sends: AtomicUsize, +} + +#[async_trait] +impl Transport for CountingTransport { + fn profile(&self) -> LinkProfile { + LinkProfile { + mtu: 64 * 1024, + latency: LatencyClass::Millis, + cost: CostClass::Metered, + broadcast: false, + } + } + + async fn reachable(&self, hint: &DeliveryHint) -> Reachability { + match hint { + DeliveryHint::MeshNode(_) => Reachability::Now, + _ => Reachability::Unreachable, + } + } + + async fn send( + &self, + _hint: &DeliveryHint, + _envelope: &Envelope, + ) -> kult_transport::Result { + self.sends.fetch_add(1, Ordering::SeqCst); + Ok(SendReceipt::HandedToLink) + } + + async fn recv(&self) -> kult_transport::Result> { + Ok(Vec::new()) + } +} + #[derive(Clone, Copy, Debug)] enum Target { + PrekeyPublish, PairwiseSend, HandshakeReceive, PairwiseReceive, @@ -24,6 +76,11 @@ enum Target { Maintenance, MaintenanceReset, MaintenanceExpiry, + GroupState, + GroupSend, + GroupReceive, + AttachmentStage, + AttachmentState, } struct Fixture { @@ -128,7 +185,6 @@ fn consume( now: u64, rng: &mut StdRng, ) -> Result { - let mut acks = Vec::new(); let mut established = false; node.consume( envelope, @@ -138,7 +194,6 @@ fn consume( }, now, rng, - &mut acks, &mut established, ) } @@ -150,6 +205,7 @@ fn run_store_case( seed: u64, ) -> bool { match target { + Target::PrekeyPublish => run_prekey_publish(point, failure, seed), Target::PairwiseSend => run_pairwise_send(point, failure, seed), Target::HandshakeReceive => run_handshake_receive(point, failure, seed), Target::PairwiseReceive => run_pairwise_receive(point, failure, seed), @@ -157,6 +213,11 @@ fn run_store_case( Target::Maintenance => run_maintenance(point, failure, seed), Target::MaintenanceReset => run_maintenance_reset(point, failure, seed), Target::MaintenanceExpiry => run_maintenance_expiry(point, failure, seed), + Target::GroupState => run_group_state(point, failure, seed), + Target::GroupSend => run_group_send(point, failure, seed), + Target::GroupReceive => run_group_receive(point, failure, seed), + Target::AttachmentStage => run_attachment_stage(point, failure, seed), + Target::AttachmentState => run_attachment_state(point, failure, seed), } } @@ -164,6 +225,34 @@ fn expected_committed(result_ok: bool, point: CommitFailpoint) -> bool { result_ok || point == CommitFailpoint::AfterCommit } +fn run_prekey_publish(point: CommitFailpoint, failure: CommitFailure, seed: u64) -> bool { + let mut fixture = Fixture::new(seed); + let before = fixture.alice.store.get_prekeys().unwrap().unwrap(); + fixture.alice.arm_commit_failpoint(point, failure); + let result = fixture.alice.handshake_bundle(NOW + 9, &mut fixture.rng); + let result_ok = result.is_ok(); + let committed = fixture.alice.store.get_prekeys().unwrap().unwrap() != before; + assert_eq!(committed, expected_committed(result_ok, point)); + + let Fixture { + _directory, + alice_path, + bob_path: _, + alice, + bob: _, + alice_id: _, + bob_id: _, + mut rng, + } = fixture; + drop(alice); + let mut alice = Node::open(&alice_path, b"alice").unwrap(); + if !committed { + alice.handshake_bundle(NOW + 10, &mut rng).unwrap(); + } + assert_ne!(alice.store.get_prekeys().unwrap().unwrap(), before); + result_ok +} + fn run_pairwise_send(point: CommitFailpoint, failure: CommitFailure, seed: u64) -> bool { let mut fixture = Fixture::new(seed); fixture.establish(); @@ -714,41 +803,85 @@ fn run_maintenance_expiry(point: CommitFailpoint, failure: CommitFailure, seed: result_ok } -fn transition_commits_when_fired(point: TransitionFailpoint) -> bool { - matches!( - point, - TransitionFailpoint::BeforeMemoryReplacement | TransitionFailpoint::AfterMemoryReplacement - ) +fn run_group_state(point: CommitFailpoint, failure: CommitFailure, seed: u64) -> bool { + let mut fixture = Fixture::new(seed); + let group = fixture + .alice + .create_group("state before", &[fixture.bob_id], &mut fixture.rng) + .unwrap(); + fixture.alice.drain_events(); + let before = fixture.alice.store.get_group(&group).unwrap().unwrap(); + let mut after = before.clone(); + after.name = "state after".to_owned(); + fixture.alice.arm_commit_failpoint(point, failure); + let result = fixture.alice.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[GroupStateTransition { + before: Some(&before), + after: Some(&after), + }], + chains: &[], + contacts: &[], + authorities: &[], + delete_controls: &[], + presentation_changed: true, + }), + &mut fixture.rng, + ); + let result_ok = result.is_ok(); + let committed = fixture.alice.store.get_group(&group).unwrap() == Some(after.clone()); + assert_eq!(committed, expected_committed(result_ok, point)); + + let Fixture { + _directory, + alice_path, + bob_path: _, + alice, + bob: _, + alice_id: _, + bob_id: _, + mut rng, + } = fixture; + drop(alice); + let alice = Node::open(&alice_path, b"alice").unwrap(); + if !committed { + alice + .store + .commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[GroupStateTransition { + before: Some(&before), + after: Some(&after), + }], + chains: &[], + contacts: &[], + authorities: &[], + delete_controls: &[], + presentation_changed: true, + }), + &mut rng, + ) + .unwrap(); + } + assert_eq!(alice.store.get_group(&group).unwrap(), Some(after)); + result_ok } -fn run_pairwise_send_transition(point: TransitionFailpoint, seed: u64) -> bool { +fn run_group_send(point: CommitFailpoint, failure: CommitFailure, seed: u64) -> bool { let mut fixture = Fixture::new(seed); fixture.establish(); - let id = [0x71; 16]; - fixture.alice.arm_transition_failpoint(point); - let result = fixture.alice.send_message_with_id( - &fixture.bob_id, - b"send checkpoint", - id, - NOW + 20, - NOW + 20, - &mut fixture.rng, - ); - let fired = fixture.alice.transition_failpoint_fired(); - let committed = fixture + let group = fixture .alice - .store - .messages_with(&fixture.bob_id) - .unwrap() - .iter() - .any(|message| message.id == id); - if fired { - assert!(result.is_err()); - assert_eq!(committed, transition_commits_when_fired(point)); - } else { - result.unwrap(); - assert!(committed); - } + .create_group("planned send", &[fixture.bob_id], &mut fixture.rng) + .unwrap(); + fixture.alice.arm_commit_failpoint(point, failure); + let result = + fixture + .alice + .group_send(&group, b"group planned send", NOW + 80, &mut fixture.rng); + let result_ok = result.is_ok(); + let committed = fixture.alice.store.group_messages(&group).unwrap().len() == 1; + assert_eq!(committed, expected_committed(result_ok, point)); assert_eq!( fixture .alice @@ -756,10 +889,11 @@ fn run_pairwise_send_transition(point: TransitionFailpoint, seed: u64) -> bool { .queue_all() .unwrap() .iter() - .filter(|(_, item)| item.msg_id == Some(id)) + .filter(|(_, item)| item.group_msg_id.is_some()) .count(), usize::from(committed) ); + let Fixture { _directory, alice_path, @@ -767,153 +901,95 @@ fn run_pairwise_send_transition(point: TransitionFailpoint, seed: u64) -> bool { alice, bob: _, alice_id: _, - bob_id, + bob_id: _, mut rng, } = fixture; drop(alice); let mut alice = Node::open(&alice_path, b"alice").unwrap(); if !committed { alice - .send_message_with_id( - &bob_id, - b"send checkpoint", - id, - NOW + 20, - NOW + 20, - &mut rng, - ) + .group_send(&group, b"group planned send", NOW + 81, &mut rng) .unwrap(); } - assert_eq!( - alice - .store - .messages_with(&bob_id) - .unwrap() - .iter() - .filter(|message| message.id == id) - .count(), - 1 - ); - fired + assert_eq!(alice.store.group_messages(&group).unwrap().len(), 1); + result_ok } -fn run_handshake_transition(point: TransitionFailpoint, seed: u64) -> bool { +fn run_group_receive(point: CommitFailpoint, failure: CommitFailure, seed: u64) -> bool { let mut fixture = Fixture::new(seed); - let id = [0x72; 16]; - fixture + fixture.establish(); + let group = fixture .alice - .send_message_with_id( - &fixture.bob_id, - b"handshake checkpoint", - id, - NOW + 20, - NOW + 20, - &mut fixture.rng, - ) + .create_group("planned receive", &[fixture.bob_id], &mut fixture.rng) .unwrap(); - let envelope = queued_message(&fixture.alice, id); - let content_id = envelope.content_id(); - let sequence = fixture - .bob + futures::executor::block_on(fixture.alice.tick_groups(NOW + 90, &mut fixture.rng)).unwrap(); + let announce = fixture + .alice .store - .pending_push(&envelope, NOW + 20, &mut fixture.rng) - .unwrap(); - fixture.bob.arm_transition_failpoint(point); - let result = consume( + .queue_all() + .unwrap() + .into_iter() + .find_map(|(_, item)| { + (item.envelope.kind == EnvelopeKind::GroupControl).then_some(item.envelope) + }) + .expect("group announce"); + consume( &mut fixture.bob, - &envelope, - Some(sequence), - NOW + 21, + &announce, + None, + NOW + 91, &mut fixture.rng, - ); - let fired = fixture.bob.transition_failpoint_fired(); - let committed = fixture.bob.store.is_seen(&content_id).unwrap(); - if fired { - assert!(result.is_err()); - assert_eq!(committed, transition_commits_when_fired(point)); - } else { - result.unwrap(); - assert!(committed); - } - assert_eq!( - pending_contains(&fixture.bob, sequence, content_id), - !committed - ); - let Fixture { - _directory, - alice_path: _, - bob_path, - alice: _, - bob, - alice_id, - bob_id: _, - mut rng, - } = fixture; - drop(bob); - let mut bob = Node::open(&bob_path, b"bob").unwrap(); - consume( - &mut bob, - &envelope, - (!committed).then_some(sequence), - NOW + 22, - &mut rng, ) .unwrap(); - assert!(bob.store.get_session(&alice_id).unwrap().is_some()); - assert_eq!(bob.store.messages_with(&alice_id).unwrap().len(), 1); - fired -} - -fn run_pairwise_receive_transition(point: TransitionFailpoint, seed: u64) -> bool { - let mut fixture = Fixture::new(seed); - fixture.establish(); - let id = [0x73; 16]; fixture + .bob + .apply_deferred_controls(NOW + 91, &mut fixture.rng) + .unwrap(); + let message_id = fixture .alice - .send_message_with_id( - &fixture.bob_id, - b"receive checkpoint", - id, - NOW + 20, - NOW + 20, - &mut fixture.rng, - ) + .group_send(&group, b"group planned receive", NOW + 92, &mut fixture.rng) .unwrap(); - let envelope = queued_message(&fixture.alice, id); + let envelope = fixture + .alice + .store + .queue_all() + .unwrap() + .into_iter() + .find_map(|(_, item)| (item.group_msg_id == Some(message_id)).then_some(item.envelope)) + .expect("group message"); let content_id = envelope.content_id(); - let sequence = fixture + let pending_sequence = fixture .bob .store - .pending_push(&envelope, NOW + 20, &mut fixture.rng) + .pending_push(&envelope, NOW + 92, &mut fixture.rng) .unwrap(); - fixture.bob.arm_transition_failpoint(point); + fixture.bob.arm_commit_failpoint(point, failure); let result = consume( &mut fixture.bob, &envelope, - Some(sequence), - NOW + 21, + Some(pending_sequence), + NOW + 93, &mut fixture.rng, ); - let fired = fixture.bob.transition_failpoint_fired(); + let result_ok = result.is_ok(); let committed = fixture.bob.store.is_seen(&content_id).unwrap(); - if fired { - assert!(result.is_err()); - assert_eq!(committed, transition_commits_when_fired(point)); - } else { - result.unwrap(); - assert!(committed); - } + assert_eq!(committed, expected_committed(result_ok, point)); assert_eq!( - pending_contains(&fixture.bob, sequence, content_id), + fixture.bob.store.group_messages(&group).unwrap().len(), + usize::from(committed) + ); + assert_eq!( + pending_contains(&fixture.bob, pending_sequence, content_id), !committed ); + let Fixture { _directory, alice_path: _, bob_path, alice: _, bob, - alice_id, + alice_id: _, bob_id: _, mut rng, } = fixture; @@ -922,57 +998,828 @@ fn run_pairwise_receive_transition(point: TransitionFailpoint, seed: u64) -> boo consume( &mut bob, &envelope, - (!committed).then_some(sequence), - NOW + 22, + (!committed).then_some(pending_sequence), + NOW + 94, &mut rng, ) .unwrap(); - assert_eq!( - bob.store - .messages_with(&alice_id) - .unwrap() - .iter() - .filter(|message| message.body == b"receive checkpoint") - .count(), - 1 - ); - fired + assert_eq!(bob.store.group_messages(&group).unwrap().len(), 1); + result_ok } -fn run_receipt_transition(point: TransitionFailpoint, seed: u64) -> bool { - let mut fixture = Fixture::new(seed); +fn attachment_records(peer: [u8; 32]) -> (MessageRecord, MediaTransferRecord, MediaObjectRecord) { + let content_id = [0xa1; 16]; + let transfer_id = [0xa2; 16]; + ( + MessageRecord { + id: content_id, + peer, + direction: Direction::Outbound, + state: DeliveryState::Queued, + timestamp: NOW + 100, + body: vec![1], + wire_id: None, + }, + MediaTransferRecord { + local_id: transfer_id, + peer, + direction: MediaDirection::Outbound, + scope: MediaScope::Pairwise, + scope_id: [0xa3; 32], + manifest_author: [0xa4; 32], + manifest_content_id: content_id, + entitled_peers: vec![peer], + state: MediaTransferState::Queued, + updated_at: NOW + 100, + }, + MediaObjectRecord { + local_id: [0xa5; 16], + transfer_id, + object_id: [0xa6; 16], + role: 0, + total_len: 1, + chunk_count: 1, + content_hash: [0xa7; 32], + media_type: "application/octet-stream".to_owned(), + filename: Some("planned.bin".to_owned()), + state: MediaTransferState::Queued, + verified_bitmap: vec![0], + chunk_addresses: vec![None], + verified_bytes: 0, + }, + ) +} + +fn run_attachment_stage(point: CommitFailpoint, failure: CommitFailure, seed: u64) -> bool { + let mut fixture = Fixture::new(seed); + let (message, transfer, object) = attachment_records(fixture.bob_id); + fixture.alice.arm_commit_failpoint(point, failure); + let result = fixture.alice.store.commit_plan( + CommitPlan::AttachmentStage(AttachmentStagePlan { + message: Some(&message), + group_message: None, + media_transfers: core::slice::from_ref(&transfer), + media_objects: core::slice::from_ref(&object), + ephemeral: None, + presentation_changed: true, + }), + &mut fixture.rng, + ); + let result_ok = result.is_ok(); + let committed = matches!( + fixture + .alice + .store + .get_media_transfer(&transfer.local_id) + .unwrap(), + Some(MediaRecord::Available(_)) + ); + assert_eq!(committed, expected_committed(result_ok, point)); + assert_eq!( + fixture + .alice + .store + .messages_with(&fixture.bob_id) + .unwrap() + .len(), + usize::from(committed) + ); + + let Fixture { + _directory, + alice_path, + bob_path: _, + alice, + bob: _, + alice_id: _, + bob_id: _, + mut rng, + } = fixture; + drop(alice); + let alice = Node::open(&alice_path, b"alice").unwrap(); + if !committed { + alice + .store + .commit_plan( + CommitPlan::AttachmentStage(AttachmentStagePlan { + message: Some(&message), + group_message: None, + media_transfers: core::slice::from_ref(&transfer), + media_objects: core::slice::from_ref(&object), + ephemeral: None, + presentation_changed: true, + }), + &mut rng, + ) + .unwrap(); + } + assert!(matches!( + alice.store.get_media_transfer(&transfer.local_id).unwrap(), + Some(MediaRecord::Available(_)) + )); + result_ok +} + +fn run_attachment_state(point: CommitFailpoint, failure: CommitFailure, seed: u64) -> bool { + let mut fixture = Fixture::new(seed); + let (_, before, _) = attachment_records(fixture.bob_id); + fixture + .alice + .store + .put_media_transfer(&before, &mut fixture.rng) + .unwrap(); + let mut after = before.clone(); + after.state = MediaTransferState::Paused; + after.updated_at += 1; + fixture.alice.arm_commit_failpoint(point, failure); + let result = fixture.alice.store.commit_plan( + CommitPlan::AttachmentState(AttachmentStatePlan { + media_transfers: &[MediaTransferTransition { + before: &before, + after: &after, + }], + media_objects: &[], + delete_controls: &[], + presentation_changed: true, + }), + &mut fixture.rng, + ); + let result_ok = result.is_ok(); + let committed = fixture + .alice + .store + .get_media_transfer(&before.local_id) + .unwrap() + == Some(MediaRecord::Available(after.clone())); + assert_eq!(committed, expected_committed(result_ok, point)); + + let Fixture { + _directory, + alice_path, + bob_path: _, + alice, + bob: _, + alice_id: _, + bob_id: _, + mut rng, + } = fixture; + drop(alice); + let alice = Node::open(&alice_path, b"alice").unwrap(); + if !committed { + alice + .store + .commit_plan( + CommitPlan::AttachmentState(AttachmentStatePlan { + media_transfers: &[MediaTransferTransition { + before: &before, + after: &after, + }], + media_objects: &[], + delete_controls: &[], + presentation_changed: true, + }), + &mut rng, + ) + .unwrap(); + } + assert_eq!( + alice.store.get_media_transfer(&before.local_id).unwrap(), + Some(MediaRecord::Available(after)) + ); + result_ok +} + +struct LargeGroupFanout { + before_group: GroupRecord, + after_group: GroupRecord, + message: GroupMessageRecord, + deliveries: Vec, + queue: Vec, +} + +impl LargeGroupFanout { + fn commit( + &self, + node: &Node, + rng: &mut impl rand_core::CryptoRngCore, + ) -> kult_store::Result { + node.store.commit_plan( + CommitPlan::GroupSend(GroupSendPlan { + group: Some(GroupTransition { + before: &self.before_group, + after: &self.after_group, + }), + message: Some(&self.message), + message_update: None, + deliveries: &self.deliveries, + delivery_updates: &[], + queue: &self.queue, + scheduled: None, + ephemeral: None, + media_transfers: &[], + delete_chains: &[], + authority: None, + presentation_changed: true, + }), + rng, + ) + } +} + +fn prepare_large_group_fanout(fixture: &mut Fixture) -> LargeGroupFanout { + let group = fixture + .alice + .create_group("bounded maximum", &[fixture.bob_id], &mut fixture.rng) + .unwrap(); + let initial = fixture.alice.store.get_group(&group).unwrap().unwrap(); + let mut before_group = initial.clone(); + let mut discriminator = 1u8; + while before_group.members.len() < 64 { + let mut peer = [0u8; 32]; + peer[0] = 0x80; + peer[1] = discriminator; + discriminator = discriminator.checked_add(1).unwrap(); + if before_group + .members + .iter() + .any(|member| member.peer == peer) + { + continue; + } + before_group.members.push(GroupMember { + peer, + identity: vec![0x49, discriminator], + }); + } + fixture + .alice + .store + .commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[GroupStateTransition { + before: Some(&initial), + after: Some(&before_group), + }], + chains: &[], + contacts: &[], + authorities: &[], + delete_controls: &[], + presentation_changed: false, + }), + &mut fixture.rng, + ) + .unwrap(); + + let mut after_group = before_group.clone(); + fixture + .alice + .rotate_group(&mut after_group, &mut fixture.rng) + .unwrap(); + let id = [0xb1; 16]; + let wire = vec![0xc7; 96]; + let mut account_deliveries = Vec::new(); + let mut deliveries = Vec::new(); + let mut queue = Vec::new(); + for (account_index, member) in before_group + .members + .iter() + .filter(|member| member.peer != before_group.creator) + .enumerate() + { + let mut first_wire = None; + for device_index in 0..8 { + let mut device = [0u8; 32]; + device[0] = 0xd0; + device[1] = u8::try_from(account_index + 1).unwrap(); + device[2] = u8::try_from(device_index + 1).unwrap(); + let envelope = Envelope::new(EnvelopeKind::GroupMessage, device, wire.clone()); + let wire_id = envelope.content_id(); + first_wire.get_or_insert(wire_id); + deliveries.push(MessageDeviceDeliveryRecord { + message: id, + account: member.peer, + device, + wire_id: Some(wire_id), + state: DeliveryState::Queued, + }); + queue.push(QueueItem { + peer: device, + msg_id: None, + group_msg_id: Some(id), + class: QueueClass::Interactive, + created_at: NOW + 110, + attempts: 0, + next_attempt_at: NOW + 110, + envelope, + }); + } + account_deliveries.push(GroupDelivery { + peer: member.peer, + wire_id: first_wire, + state: DeliveryState::Queued, + }); + } + assert_eq!(account_deliveries.len(), 63); + assert_eq!(deliveries.len(), 504); + assert_eq!(queue.len(), 504); + LargeGroupFanout { + before_group, + after_group, + message: GroupMessageRecord { + id, + group, + sender: initial.creator, + direction: Direction::Outbound, + timestamp: NOW + 110, + body: b"maximum bounded group fan-out".to_vec(), + deliveries: account_deliveries, + wire_body: None, + }, + deliveries, + queue, + } +} + +fn transition_commits_when_fired(point: TransitionFailpoint) -> bool { + matches!( + point, + TransitionFailpoint::BeforeMemoryReplacement | TransitionFailpoint::AfterMemoryReplacement + ) +} + +fn run_pairwise_send_transition(point: TransitionFailpoint, seed: u64) -> bool { + let mut fixture = Fixture::new(seed); + fixture.establish(); + let id = [0x71; 16]; + fixture.alice.arm_transition_failpoint(point); + let result = fixture.alice.send_message_with_id( + &fixture.bob_id, + b"send checkpoint", + id, + NOW + 20, + NOW + 20, + &mut fixture.rng, + ); + let fired = fixture.alice.transition_failpoint_fired(); + let committed = fixture + .alice + .store + .messages_with(&fixture.bob_id) + .unwrap() + .iter() + .any(|message| message.id == id); + if fired { + assert!(result.is_err()); + assert_eq!(committed, transition_commits_when_fired(point)); + } else { + result.unwrap(); + assert!(committed); + } + assert_eq!( + fixture + .alice + .store + .queue_all() + .unwrap() + .iter() + .filter(|(_, item)| item.msg_id == Some(id)) + .count(), + usize::from(committed) + ); + let Fixture { + _directory, + alice_path, + bob_path: _, + alice, + bob: _, + alice_id: _, + bob_id, + mut rng, + } = fixture; + drop(alice); + let mut alice = Node::open(&alice_path, b"alice").unwrap(); + if !committed { + alice + .send_message_with_id( + &bob_id, + b"send checkpoint", + id, + NOW + 20, + NOW + 20, + &mut rng, + ) + .unwrap(); + } + assert_eq!( + alice + .store + .messages_with(&bob_id) + .unwrap() + .iter() + .filter(|message| message.id == id) + .count(), + 1 + ); + fired +} + +fn run_handshake_transition(point: TransitionFailpoint, seed: u64) -> bool { + let mut fixture = Fixture::new(seed); + let id = [0x72; 16]; + fixture + .alice + .send_message_with_id( + &fixture.bob_id, + b"handshake checkpoint", + id, + NOW + 20, + NOW + 20, + &mut fixture.rng, + ) + .unwrap(); + let envelope = queued_message(&fixture.alice, id); + let content_id = envelope.content_id(); + let sequence = fixture + .bob + .store + .pending_push(&envelope, NOW + 20, &mut fixture.rng) + .unwrap(); + fixture.bob.arm_transition_failpoint(point); + let result = consume( + &mut fixture.bob, + &envelope, + Some(sequence), + NOW + 21, + &mut fixture.rng, + ); + let fired = fixture.bob.transition_failpoint_fired(); + let committed = fixture.bob.store.is_seen(&content_id).unwrap(); + if fired { + assert!(result.is_err()); + assert_eq!(committed, transition_commits_when_fired(point)); + } else { + result.unwrap(); + assert!(committed); + } + assert_eq!( + pending_contains(&fixture.bob, sequence, content_id), + !committed + ); + let Fixture { + _directory, + alice_path: _, + bob_path, + alice: _, + bob, + alice_id, + bob_id: _, + mut rng, + } = fixture; + drop(bob); + let mut bob = Node::open(&bob_path, b"bob").unwrap(); + consume( + &mut bob, + &envelope, + (!committed).then_some(sequence), + NOW + 22, + &mut rng, + ) + .unwrap(); + assert!(bob.store.get_session(&alice_id).unwrap().is_some()); + assert_eq!(bob.store.messages_with(&alice_id).unwrap().len(), 1); + fired +} + +fn run_pairwise_receive_transition(point: TransitionFailpoint, seed: u64) -> bool { + let mut fixture = Fixture::new(seed); + fixture.establish(); + let id = [0x73; 16]; + fixture + .alice + .send_message_with_id( + &fixture.bob_id, + b"receive checkpoint", + id, + NOW + 20, + NOW + 20, + &mut fixture.rng, + ) + .unwrap(); + let envelope = queued_message(&fixture.alice, id); + let content_id = envelope.content_id(); + let sequence = fixture + .bob + .store + .pending_push(&envelope, NOW + 20, &mut fixture.rng) + .unwrap(); + fixture.bob.arm_transition_failpoint(point); + let result = consume( + &mut fixture.bob, + &envelope, + Some(sequence), + NOW + 21, + &mut fixture.rng, + ); + let fired = fixture.bob.transition_failpoint_fired(); + let committed = fixture.bob.store.is_seen(&content_id).unwrap(); + if fired { + assert!(result.is_err()); + assert_eq!(committed, transition_commits_when_fired(point)); + } else { + result.unwrap(); + assert!(committed); + } + assert_eq!( + pending_contains(&fixture.bob, sequence, content_id), + !committed + ); + let Fixture { + _directory, + alice_path: _, + bob_path, + alice: _, + bob, + alice_id, + bob_id: _, + mut rng, + } = fixture; + drop(bob); + let mut bob = Node::open(&bob_path, b"bob").unwrap(); + consume( + &mut bob, + &envelope, + (!committed).then_some(sequence), + NOW + 22, + &mut rng, + ) + .unwrap(); + assert_eq!( + bob.store + .messages_with(&alice_id) + .unwrap() + .iter() + .filter(|message| message.body == b"receive checkpoint") + .count(), + 1 + ); + fired +} + +fn run_receipt_transition(point: TransitionFailpoint, seed: u64) -> bool { + let mut fixture = Fixture::new(seed); + fixture.establish(); + let id = [0x74; 16]; + fixture + .alice + .send_message_with_id( + &fixture.bob_id, + b"receipt checkpoint", + id, + NOW + 20, + NOW + 20, + &mut fixture.rng, + ) + .unwrap(); + let message = queued_message(&fixture.alice, id); + consume(&mut fixture.bob, &message, None, NOW + 21, &mut fixture.rng).unwrap(); + let (_, receipt) = queued_control(&fixture.bob); + let content_id = receipt.content_id(); + let sequence = fixture + .alice + .store + .pending_push(&receipt, NOW + 21, &mut fixture.rng) + .unwrap(); + fixture.alice.arm_transition_failpoint(point); + let result = consume( + &mut fixture.alice, + &receipt, + Some(sequence), + NOW + 22, + &mut fixture.rng, + ); + let fired = fixture.alice.transition_failpoint_fired(); + let committed = fixture.alice.store.is_seen(&content_id).unwrap(); + if fired { + assert!(result.is_err()); + assert_eq!(committed, transition_commits_when_fired(point)); + } else { + result.unwrap(); + assert!(committed); + } + assert_eq!( + pending_contains(&fixture.alice, sequence, content_id), + !committed + ); + let Fixture { + _directory, + alice_path, + bob_path: _, + alice, + bob: _, + alice_id: _, + bob_id, + mut rng, + } = fixture; + drop(alice); + let mut alice = Node::open(&alice_path, b"alice").unwrap(); + consume( + &mut alice, + &receipt, + (!committed).then_some(sequence), + NOW + 23, + &mut rng, + ) + .unwrap(); + assert_eq!( + alice + .store + .messages_with(&bob_id) + .unwrap() + .into_iter() + .find(|message| message.id == id) + .unwrap() + .state, + DeliveryState::Delivered + ); + fired +} + +fn run_group_send_transition(point: TransitionFailpoint, seed: u64) -> bool { + let mut fixture = Fixture::new(seed); + fixture.establish(); + let group = fixture + .alice + .create_group("group send checkpoint", &[fixture.bob_id], &mut fixture.rng) + .unwrap(); + let id = [0x79; 16]; + fixture.alice.arm_transition_failpoint(point); + let result = fixture.alice.group_send_with_id( + &group, + b"group send checkpoint", + id, + NOW + 24, + NOW + 24, + &mut fixture.rng, + ); + let fired = fixture.alice.transition_failpoint_fired(); + let committed = fixture + .alice + .store + .group_messages(&group) + .unwrap() + .iter() + .any(|message| message.id == id); + if fired { + assert!(result.is_err()); + assert_eq!(committed, transition_commits_when_fired(point)); + } else { + result.unwrap(); + assert!(committed); + } + let Fixture { + _directory, + alice_path, + bob_path: _, + alice, + bob: _, + alice_id: _, + bob_id: _, + mut rng, + } = fixture; + drop(alice); + let mut alice = Node::open(&alice_path, b"alice").unwrap(); + if !committed { + alice + .group_send_with_id( + &group, + b"group send checkpoint", + id, + NOW + 24, + NOW + 24, + &mut rng, + ) + .unwrap(); + } + assert_eq!( + alice + .store + .group_messages(&group) + .unwrap() + .iter() + .filter(|message| message.id == id) + .count(), + 1 + ); + fired +} + +fn run_prekey_publish_transition(point: TransitionFailpoint, seed: u64) -> bool { + let mut fixture = Fixture::new(seed); + let before = fixture.alice.store.get_prekeys().unwrap().unwrap(); + fixture.alice.arm_transition_failpoint(point); + let result = fixture.alice.handshake_bundle(NOW + 23, &mut fixture.rng); + let fired = fixture.alice.transition_failpoint_fired(); + let committed = fixture.alice.store.get_prekeys().unwrap().unwrap() != before; + if fired { + assert!(result.is_err()); + assert_eq!(committed, transition_commits_when_fired(point)); + } else { + result.unwrap(); + assert!(committed); + } + let Fixture { + _directory, + alice_path, + bob_path: _, + alice, + bob: _, + alice_id: _, + bob_id: _, + mut rng, + } = fixture; + drop(alice); + let mut alice = Node::open(&alice_path, b"alice").unwrap(); + if !committed { + alice.handshake_bundle(NOW + 24, &mut rng).unwrap(); + } + assert_ne!(alice.store.get_prekeys().unwrap().unwrap(), before); + assert_eq!( + alice.vault.encode(), + alice.store.get_prekeys().unwrap().unwrap() + ); + fired +} + +fn run_group_receive_transition(point: TransitionFailpoint, seed: u64) -> bool { + let mut fixture = Fixture::new(seed); fixture.establish(); - let id = [0x74; 16]; + let group = fixture + .alice + .create_group( + "group receive checkpoint", + &[fixture.bob_id], + &mut fixture.rng, + ) + .unwrap(); + futures::executor::block_on(fixture.alice.tick_groups(NOW + 25, &mut fixture.rng)).unwrap(); + let announce = fixture + .alice + .store + .queue_all() + .unwrap() + .into_iter() + .find_map(|(_, item)| { + (item.envelope.kind == EnvelopeKind::GroupControl).then_some(item.envelope) + }) + .expect("group announce"); + consume( + &mut fixture.bob, + &announce, + None, + NOW + 26, + &mut fixture.rng, + ) + .unwrap(); + fixture + .bob + .apply_deferred_controls(NOW + 26, &mut fixture.rng) + .unwrap(); + let id = [0x7a; 16]; fixture .alice - .send_message_with_id( - &fixture.bob_id, - b"receipt checkpoint", + .group_send_with_id( + &group, + b"group receive checkpoint", id, - NOW + 20, - NOW + 20, + NOW + 27, + NOW + 27, &mut fixture.rng, ) .unwrap(); - let message = queued_message(&fixture.alice, id); - consume(&mut fixture.bob, &message, None, NOW + 21, &mut fixture.rng).unwrap(); - let (_, receipt) = queued_control(&fixture.bob); - let content_id = receipt.content_id(); - let sequence = fixture + let envelope = fixture .alice .store - .pending_push(&receipt, NOW + 21, &mut fixture.rng) + .queue_all() + .unwrap() + .into_iter() + .find_map(|(_, item)| (item.group_msg_id == Some(id)).then_some(item.envelope)) + .expect("group message"); + let content_id = envelope.content_id(); + let sequence = fixture + .bob + .store + .pending_push(&envelope, NOW + 27, &mut fixture.rng) .unwrap(); - fixture.alice.arm_transition_failpoint(point); + fixture.bob.arm_transition_failpoint(point); let result = consume( - &mut fixture.alice, - &receipt, + &mut fixture.bob, + &envelope, Some(sequence), - NOW + 22, + NOW + 28, &mut fixture.rng, ); - let fired = fixture.alice.transition_failpoint_fired(); - let committed = fixture.alice.store.is_seen(&content_id).unwrap(); + let fired = fixture.bob.transition_failpoint_fired(); + let committed = fixture.bob.store.is_seen(&content_id).unwrap(); if fired { assert!(result.is_err()); assert_eq!(committed, transition_commits_when_fired(point)); @@ -981,39 +1828,42 @@ fn run_receipt_transition(point: TransitionFailpoint, seed: u64) -> bool { assert!(committed); } assert_eq!( - pending_contains(&fixture.alice, sequence, content_id), + pending_contains(&fixture.bob, sequence, content_id), !committed ); let Fixture { _directory, - alice_path, - bob_path: _, - alice, - bob: _, + alice_path: _, + bob_path, + alice: _, + bob, alice_id: _, - bob_id, + bob_id: _, mut rng, } = fixture; - drop(alice); - let mut alice = Node::open(&alice_path, b"alice").unwrap(); - consume( - &mut alice, - &receipt, + drop(bob); + let mut bob = Node::open(&bob_path, b"bob").unwrap(); + let retry = consume( + &mut bob, + &envelope, (!committed).then_some(sequence), - NOW + 23, + NOW + 29, &mut rng, ) .unwrap(); + assert!( + matches!(retry, Consumed::Done | Consumed::DoneAtomic), + "group receive retry remained deferred at {point:?}" + ); assert_eq!( - alice - .store - .messages_with(&bob_id) - .unwrap() - .into_iter() - .find(|message| message.id == id) + bob.store + .group_messages(&group) .unwrap() - .state, - DeliveryState::Delivered + .iter() + .filter(|message| message.body == b"group receive checkpoint") + .count(), + 1, + "group receive restart did not converge at {point:?}; fired={fired}, committed={committed}" ); fired } @@ -1061,6 +1911,7 @@ fn run_maintenance_reset_transition(point: TransitionFailpoint, seed: u64) -> bo #[test] fn every_transaction_statement_is_all_or_nothing_after_restart() { let targets = [ + Target::PrekeyPublish, Target::PairwiseSend, Target::HandshakeReceive, Target::PairwiseReceive, @@ -1068,6 +1919,11 @@ fn every_transaction_statement_is_all_or_nothing_after_restart() { Target::Maintenance, Target::MaintenanceReset, Target::MaintenanceExpiry, + Target::GroupState, + Target::GroupSend, + Target::GroupReceive, + Target::AttachmentStage, + Target::AttachmentState, ]; let mut seed = 0xa280_0000; for target in targets { @@ -1104,17 +1960,201 @@ fn every_transaction_statement_is_all_or_nothing_after_restart() { } } +#[test] +fn maximum_group_fanout_is_bounded_and_restart_atomic() { + let points = [ + CommitFailpoint::BeforeStatement(0), + CommitFailpoint::AfterStatement(505), + CommitFailpoint::AfterStatement(1009), + CommitFailpoint::BeforeCommit, + CommitFailpoint::AfterCommit, + ]; + for (offset, point) in points.into_iter().enumerate() { + let mut fixture = Fixture::new(0xa280_4000 + offset as u64); + let fanout = prepare_large_group_fanout(&mut fixture); + fixture + .alice + .arm_commit_failpoint(point, CommitFailure::Interrupted); + let result = fanout.commit(&fixture.alice, &mut fixture.rng); + assert!(result.is_err(), "large fan-out failpoint was not reached"); + let committed = fixture + .alice + .store + .group_messages(&fanout.message.group) + .unwrap() + .iter() + .any(|message| message.id == fanout.message.id); + assert_eq!(committed, point == CommitFailpoint::AfterCommit); + + let Fixture { + _directory, + alice_path, + bob_path: _, + alice, + bob: _, + alice_id: _, + bob_id: _, + mut rng, + } = fixture; + drop(alice); + let alice = Node::open(&alice_path, b"alice").unwrap(); + if !committed { + fanout.commit(&alice, &mut rng).unwrap(); + } + assert_eq!( + alice + .store + .message_device_deliveries(&fanout.message.id) + .unwrap() + .len(), + 504 + ); + assert_eq!( + alice + .store + .queue_all() + .unwrap() + .iter() + .filter(|(_, item)| item.group_msg_id == Some(fanout.message.id)) + .count(), + 504 + ); + assert_eq!( + alice + .store + .group_messages(&fanout.message.group) + .unwrap() + .iter() + .filter(|message| message.id == fanout.message.id) + .count(), + 1 + ); + } +} + +#[test] +fn group_fanout_rejects_a_ninth_device_for_one_account() { + let mut fixture = Fixture::new(0xa280_5000); + let mut fanout = prepare_large_group_fanout(&mut fixture); + let account = fanout.message.deliveries[0].peer; + let mut device = [0u8; 32]; + device[0] = 0xd0; + device[1] = 1; + device[2] = 9; + let envelope = Envelope::new(EnvelopeKind::GroupMessage, device, vec![0xc7; 96]); + let wire_id = envelope.content_id(); + fanout.deliveries.push(MessageDeviceDeliveryRecord { + message: fanout.message.id, + account, + device, + wire_id: Some(wire_id), + state: DeliveryState::Queued, + }); + fanout.queue.push(QueueItem { + peer: device, + msg_id: None, + group_msg_id: Some(fanout.message.id), + class: QueueClass::Interactive, + created_at: NOW + 110, + attempts: 0, + next_attempt_at: NOW + 110, + envelope, + }); + assert!(matches!( + fanout.commit(&fixture.alice, &mut fixture.rng), + Err(kult_store::StoreError::InvalidTransition) + )); + assert!(fixture + .alice + .store + .group_messages(&fanout.message.group) + .unwrap() + .is_empty()); +} + +#[test] +fn stable_protocol_modules_cannot_call_raw_state_setters() { + let source_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let forbidden = [ + "put_session", + "delete_session", + "put_group", + "delete_group", + "put_group_authority", + "put_group_chain", + "delete_group_chain", + "put_group_message", + "update_group_message", + "delete_group_message_record", + "put_ephemeral_record", + "put_media_transfer", + "set_media_transfer_state", + "put_media_object", + "set_media_object_state", + "commit_media_chunk", + "mark_media_complete", + "delete_media_transfer", + "delete_media_transfer_with_objects", + "delete_media_object", + "mark_seen", + "put_receipt_replay", + "put_message", + "update_message", + "delete_message_record", + "put_message_device_delivery", + "queue_push", + "queue_ack", + "queue_update", + "queue_remove_peer", + "queue_retarget_peer", + "queue_remove_message", + "queue_remove_group_message", + "queue_remove_envelope", + "pending_ack", + ]; + for entry in std::fs::read_dir(source_root).unwrap() { + let path = entry.unwrap().path(); + if path.extension().and_then(|value| value.to_str()) != Some("rs") { + continue; + } + let filename = path.file_name().and_then(|value| value.to_str()).unwrap(); + if matches!(filename, "atomic_tests.rs" | "devices.rs") { + // C2 remains an explicit deferred transaction redesign under ADR-0026. + continue; + } + let source = std::fs::read_to_string(&path).unwrap(); + let production = source.split("#[cfg(test)]").next().unwrap_or(&source); + let normalized = production.split_whitespace().collect::(); + assert!( + !normalized.contains("#[cfg(any())]"), + "{} retains disabled protocol code", + path.display() + ); + for setter in forbidden { + let needle = format!(".{setter}("); + assert!( + !normalized.contains(&needle), + "{} bypasses a typed transition through {setter}", + path.display() + ); + } + } +} + #[test] fn every_candidate_crypto_and_memory_checkpoint_has_a_binary_restart_state() { - let runners: [fn(TransitionFailpoint, u64) -> bool; 5] = [ + let crypto_runners: [fn(TransitionFailpoint, u64) -> bool; 8] = [ + run_prekey_publish_transition, run_pairwise_send_transition, run_handshake_transition, run_pairwise_receive_transition, run_receipt_transition, + run_group_send_transition, + run_group_receive_transition, run_maintenance_reset_transition, ]; let mut seed = 0xa281_0000; - for runner in runners { + for runner in crypto_runners { for before in [true, false] { let mut reached_end = false; for step in 0..16 { @@ -1131,6 +2171,17 @@ fn every_candidate_crypto_and_memory_checkpoint_has_a_binary_restart_state() { } assert!(reached_end, "crypto checkpoint test bound exceeded"); } + } + let memory_runners: [fn(TransitionFailpoint, u64) -> bool; 7] = [ + run_prekey_publish_transition, + run_pairwise_send_transition, + run_handshake_transition, + run_pairwise_receive_transition, + run_receipt_transition, + run_group_receive_transition, + run_maintenance_reset_transition, + ]; + for runner in memory_runners { for point in [ TransitionFailpoint::BeforeMemoryReplacement, TransitionFailpoint::AfterMemoryReplacement, @@ -1141,6 +2192,102 @@ fn every_candidate_crypto_and_memory_checkpoint_has_a_binary_restart_state() { } } +#[test] +fn scheduled_activation_commits_before_transport_or_presentation() { + for (offset, point) in [CommitFailpoint::BeforeCommit, CommitFailpoint::AfterCommit] + .into_iter() + .enumerate() + { + let mut fixture = Fixture::new(0xa281_8000 + offset as u64); + fixture.establish(); + fixture + .alice + .acknowledge_presentation(&mut fixture.rng) + .unwrap(); + let transport = Arc::new(CountingTransport::default()); + fixture.alice.add_transport(transport.clone()); + fixture + .alice + .set_hints( + &fixture.bob_id, + &[DeliveryHint::MeshNode(9)], + &mut fixture.rng, + ) + .unwrap(); + fixture.alice.capabilities_advertised.insert(fixture.bob_id); + let id = fixture + .alice + .schedule_message( + &fixture.bob_id, + b"scheduled commit boundary", + NOW + 50, + NOW + 40, + &mut fixture.rng, + ) + .unwrap(); + fixture.alice.drain_events(); + fixture + .alice + .arm_commit_failpoint(point, CommitFailure::Interrupted); + let result = futures::executor::block_on(fixture.alice.tick(NOW + 50, &mut fixture.rng)); + result.unwrap(); + assert_eq!( + transport.sends.load(Ordering::SeqCst) > 0, + point == CommitFailpoint::AfterCommit + ); + assert_eq!( + fixture + .alice + .store + .get_scheduled_message(&id) + .unwrap() + .is_none(), + point == CommitFailpoint::AfterCommit, + "scheduled activation failpoint was consumed by a different transition" + ); + assert_eq!( + fixture + .alice + .store + .presentation_resync_marker() + .unwrap() + .is_some(), + point == CommitFailpoint::AfterCommit + ); + assert!(!fixture.alice.drain_events().iter().any( + |event| matches!(event, Event::ScheduledMessageActivated { id: event } if *event == id) + )); + + let Fixture { + _directory, + alice_path, + bob_path: _, + alice, + bob: _, + alice_id: _, + bob_id: _, + mut rng, + } = fixture; + drop(alice); + let mut alice = Node::open(&alice_path, b"alice").unwrap(); + alice.add_transport(transport.clone()); + let reopen_events = alice.drain_events(); + if point == CommitFailpoint::AfterCommit { + assert!(reopen_events + .iter() + .any(|event| matches!(event, Event::StateResyncRequired))); + } + let events = futures::executor::block_on(alice.tick(NOW + 51, &mut rng)).unwrap(); + assert!(transport.sends.load(Ordering::SeqCst) > 0); + if point == CommitFailpoint::BeforeCommit { + assert!(events.iter().any( + |event| matches!(event, Event::ScheduledMessageActivated { id: event } if *event == id) + )); + } + assert!(alice.store.get_scheduled_message(&id).unwrap().is_none()); + } +} + #[test] fn presentation_checkpoint_recovers_with_a_deterministic_resync() { for (offset, point) in [ @@ -1296,6 +2443,7 @@ fn reordered_deferred_and_duplicate_input_converges_after_restart() { #[test] fn disk_constraint_and_duplicate_failures_leave_retryable_inputs() { let targets = [ + Target::PrekeyPublish, Target::PairwiseSend, Target::HandshakeReceive, Target::PairwiseReceive, @@ -1303,6 +2451,11 @@ fn disk_constraint_and_duplicate_failures_leave_retryable_inputs() { Target::Maintenance, Target::MaintenanceReset, Target::MaintenanceExpiry, + Target::GroupState, + Target::GroupSend, + Target::GroupReceive, + Target::AttachmentStage, + Target::AttachmentState, ]; let failures = [ CommitFailure::DiskFull, diff --git a/crates/kult-node/src/attachment.rs b/crates/kult-node/src/attachment.rs index 1490b8d..c96be00 100644 --- a/crates/kult-node/src/attachment.rs +++ b/crates/kult-node/src/attachment.rs @@ -26,12 +26,13 @@ use kult_protocol::{ MIN_EPHEMERAL_LIFETIME_SECS, }; use kult_store::{ - DeliveryState, Direction, EphemeralConversation, EphemeralMode, EphemeralRecord, - EphemeralState, EphemeralTransition, GroupDelivery, GroupMessageDelete, GroupMessageRecord, - MaintenancePlan, MediaDelete, MediaDirection, MediaObjectRecord, MediaRecord, MediaScope, - MediaTransferRecord, MediaTransferState, MessageDelete, MessageDeviceDeliveryRecord, - MessageRecord, MessageTransition, PairwiseSendPlan, QueueClass, QueueDelete, QueueItem, - SessionTransition, Store, StoreError, + AttachmentStagePlan, AttachmentStatePlan, DeferredControlRecord, DeliveryState, Direction, + EphemeralConversation, EphemeralMode, EphemeralRecord, EphemeralState, EphemeralTransition, + GroupDelivery, GroupMessageDelete, GroupMessageRecord, MaintenancePlan, MediaDelete, + MediaDirection, MediaObjectRecord, MediaObjectTransition, MediaRecord, MediaScope, + MediaTransferRecord, MediaTransferState, MediaTransferTransition, MessageDelete, + MessageDeviceDeliveryRecord, MessageRecord, MessageTransition, PairwiseSendPlan, QueueClass, + QueueDelete, QueueItem, SessionTransition, Store, StoreError, }; const BULK_CONTROL_PADDING_FLOOR: usize = 4096; @@ -129,6 +130,47 @@ fn media_object_record( } } +fn media_object_with_state( + before: &MediaObjectRecord, + state: MediaTransferState, +) -> MediaObjectRecord { + let mut after = before.clone(); + after.state = state; + if matches!( + state, + MediaTransferState::Rejected | MediaTransferState::Cancelled | MediaTransferState::Corrupt + ) { + after.chunk_addresses.fill(None); + after.verified_bitmap.fill(0); + after.verified_bytes = 0; + } + after +} + +fn commit_media_object_pairs( + store: &Store, + pairs: &[(MediaObjectRecord, MediaObjectRecord)], + rng: &mut impl CryptoRngCore, +) -> Result<()> { + if pairs.is_empty() { + return Ok(()); + } + let objects = pairs + .iter() + .map(|(before, after)| MediaObjectTransition { before, after }) + .collect::>(); + store.commit_plan( + CommitPlan::AttachmentState(AttachmentStatePlan { + media_transfers: &[], + media_objects: &objects, + delete_controls: &[], + presentation_changed: false, + }), + rng, + )?; + Ok(()) +} + fn import_object( store: &mut Store, source: &mut R, @@ -146,9 +188,13 @@ fn import_object( .map_err(|_| NodeError::InvalidAttachment)?; source.read_exact(&mut buffer[..actual_len])?; let sealed = seal_attachment_chunk(attachment_key, &context, index, &buffer[..actual_len])?; - store.commit_media_chunk(local_object_id, index, &sealed, rng)?; + let staged = store.stage_media_chunk(local_object_id, index, &sealed, rng)?; + if staged.before != staged.after { + commit_media_object_pairs(store, &[(staged.before, staged.after)], rng)?; + } } - store.mark_media_complete(local_object_id, &context.content_hash, rng)?; + let complete = store.prepare_media_complete(local_object_id, &context.content_hash)?; + commit_media_object_pairs(store, &[complete], rng)?; Ok(()) } @@ -169,13 +215,20 @@ fn import_group_object( .map_err(|_| NodeError::InvalidAttachment)?; source.read_exact(&mut buffer[..actual_len])?; let sealed = seal_attachment_chunk(attachment_key, &context, index, &buffer[..actual_len])?; + let mut pairs = Vec::new(); for local_object_id in local_object_ids { - store.commit_media_chunk(local_object_id, index, &sealed, rng)?; + let staged = store.stage_media_chunk(local_object_id, index, &sealed, rng)?; + if staged.before != staged.after { + pairs.push((staged.before, staged.after)); + } } + commit_media_object_pairs(store, &pairs, rng)?; } + let mut complete = Vec::new(); for local_object_id in local_object_ids { - store.mark_media_complete(local_object_id, &context.content_hash, rng)?; + complete.push(store.prepare_media_complete(local_object_id, &context.content_hash)?); } + commit_media_object_pairs(store, &complete, rng)?; Ok(()) } @@ -379,35 +432,49 @@ impl Node { metadata, ); - self.store.put_message( - &MessageRecord { - id: content_id, - peer: *peer, - direction: Direction::Outbound, - state: DeliveryState::Queued, - timestamp: now, - body: frame, - wire_id: None, - }, - rng, - )?; - self.store.put_media_transfer(&transfer, rng)?; - self.store.put_media_object(&object, rng)?; - if let (Some((preview_metadata, _)), Some(details)) = - (preview.as_ref(), preview_details.as_ref()) - { - self.store.put_media_object( - &media_object_record( + let message = MessageRecord { + id: content_id, + peer: *peer, + direction: Direction::Outbound, + state: DeliveryState::Queued, + timestamp: now, + body: frame, + wire_id: None, + }; + let preview_object = preview.as_ref().zip(preview_details.as_ref()).map( + |((preview_metadata, _), details)| { + media_object_record( preview_local_object_id, transfer_id, preview_object_id, AttachmentRole::Preview, details, preview_metadata, - ), - rng, - )?; - } + ) + }, + ); + let mut media_objects = vec![object.clone()]; + media_objects.extend(preview_object.iter().cloned()); + let ephemeral = expires_at.map(|expires_at| EphemeralRecord { + conversation: EphemeralConversation::Pairwise(*peer), + author: me, + content_id, + expires_at, + mode: EphemeralMode::ViewOnceAttachment, + state: EphemeralState::Active, + transfer_ids: vec![transfer_id], + }); + let receipt = self.store.commit_plan( + CommitPlan::AttachmentStage(AttachmentStagePlan { + message: Some(&message), + group_message: None, + media_transfers: core::slice::from_ref(&transfer), + media_objects: &media_objects, + ephemeral: ephemeral.as_ref(), + presentation_changed: true, + }), + rng, + )?; import_object( &mut self.store, @@ -447,20 +514,7 @@ impl Node { rng, )?; } - if let Some(expires_at) = expires_at { - self.store.put_ephemeral_record( - &EphemeralRecord { - conversation: EphemeralConversation::Pairwise(*peer), - author: me, - content_id, - expires_at, - mode: EphemeralMode::ViewOnceAttachment, - state: EphemeralState::Active, - transfer_ids: vec![transfer_id], - }, - rng, - )?; - } + self.accept_commit_receipt(receipt, []); self.emit_attachment_update(&transfer_id)?; Ok(content_id) } @@ -639,26 +693,23 @@ impl Node { None => encode_attachment(content_id, &manifest) .map_err(|_| NodeError::InvalidAttachment)?, }; - self.store.put_group_message( - &GroupMessageRecord { - id: content_id, - group: *group, - sender: me, - direction: Direction::Outbound, - timestamp: now, - body: frame, - deliveries: peers - .iter() - .map(|peer| GroupDelivery { - peer: *peer, - wire_id: None, - state: DeliveryState::Queued, - }) - .collect(), - wire_body: None, - }, - rng, - )?; + let message = GroupMessageRecord { + id: content_id, + group: *group, + sender: me, + direction: Direction::Outbound, + timestamp: now, + body: frame, + deliveries: peers + .iter() + .map(|peer| GroupDelivery { + peer: *peer, + wire_id: None, + state: DeliveryState::Queued, + }) + .collect(), + wire_body: None, + }; let primary_context = AttachmentChunkContext { scope: AttachmentChunkScope::Group, @@ -711,13 +762,38 @@ impl Node { ) }, ); - self.store.put_media_transfer(&transfer, rng)?; - self.store.put_media_object(&object, rng)?; - if let Some(preview_object) = &preview_object { - self.store.put_media_object(preview_object, rng)?; - } rows.push((transfer, object, preview_object)); } + let transfers = rows + .iter() + .map(|(transfer, _, _)| transfer.clone()) + .collect::>(); + let objects = rows + .iter() + .flat_map(|(_, object, preview)| { + core::iter::once(object.clone()).chain(preview.iter().cloned()) + }) + .collect::>(); + let ephemeral = expires_at.map(|expires_at| EphemeralRecord { + conversation: EphemeralConversation::Group(*group), + author: me, + content_id, + expires_at, + mode: EphemeralMode::ViewOnceAttachment, + state: EphemeralState::Active, + transfer_ids: transfers.iter().map(|transfer| transfer.local_id).collect(), + }); + let receipt = self.store.commit_plan( + CommitPlan::AttachmentStage(AttachmentStagePlan { + message: None, + group_message: Some(&message), + media_transfers: &transfers, + media_objects: &objects, + ephemeral: ephemeral.as_ref(), + presentation_changed: true, + }), + rng, + )?; let primary_ids = rows .iter() @@ -755,23 +831,7 @@ impl Node { rng, )?; } - if let Some(expires_at) = expires_at { - self.store.put_ephemeral_record( - &EphemeralRecord { - conversation: EphemeralConversation::Group(*group), - author: me, - content_id, - expires_at, - mode: EphemeralMode::ViewOnceAttachment, - state: EphemeralState::Active, - transfer_ids: rows - .iter() - .map(|(transfer, _, _)| transfer.local_id) - .collect(), - }, - rng, - )?; - } + self.accept_commit_receipt(receipt, []); for (transfer, _, _) in &rows { self.emit_attachment_update(&transfer.local_id)?; } @@ -1064,29 +1124,29 @@ impl Node { return Err(NodeError::InvalidAttachment); } let mut all_empty = true; - for object in self.store.media_objects_for_transfer(transfer_id)? { - if object.chunk_count == 0 { - self.store - .mark_media_complete(&object.local_id, &object.content_hash, rng)?; - } else { - all_empty = false; - self.store.set_media_object_state( - &object.local_id, - MediaTransferState::Queued, - rng, - )?; - } - } - self.store.set_media_transfer_state( - transfer_id, - if all_empty { - MediaTransferState::Complete - } else { - MediaTransferState::Queued - }, - now, - rng, - )?; + let object_pairs = self + .store + .media_objects_for_transfer(transfer_id)? + .into_iter() + .map(|object| { + let state = if object.chunk_count == 0 { + MediaTransferState::Complete + } else { + all_empty = false; + MediaTransferState::Queued + }; + let after = media_object_with_state(&object, state); + (object, after) + }) + .collect::>(); + let mut transfer_after = transfer.clone(); + transfer_after.state = if all_empty { + MediaTransferState::Complete + } else { + MediaTransferState::Queued + }; + transfer_after.updated_at = now; + self.commit_attachment_state(&transfer, &transfer_after, &object_pairs, &[], rng)?; self.emit_attachment_update(transfer_id) } @@ -1130,19 +1190,23 @@ impl Node { ) { return Err(NodeError::InvalidAttachment); } - if transfer.direction == MediaDirection::Inbound { - for object in self.store.media_objects_for_transfer(transfer_id)? { - if object.state != MediaTransferState::Complete { - self.store.set_media_object_state( - &object.local_id, - MediaTransferState::Paused, - rng, - )?; - } - } - } - self.store - .set_media_transfer_state(transfer_id, MediaTransferState::Paused, now, rng)?; + let object_pairs = if transfer.direction == MediaDirection::Inbound { + self.store + .media_objects_for_transfer(transfer_id)? + .into_iter() + .filter(|object| object.state != MediaTransferState::Complete) + .map(|object| { + let after = media_object_with_state(&object, MediaTransferState::Paused); + (object, after) + }) + .collect::>() + } else { + Vec::new() + }; + let mut transfer_after = transfer.clone(); + transfer_after.state = MediaTransferState::Paused; + transfer_after.updated_at = now; + self.commit_attachment_state(&transfer, &transfer_after, &object_pairs, &[], rng)?; self.attachment_request_at.remove(transfer_id); self.attachment_request_target.remove(transfer_id); self.emit_attachment_update(transfer_id) @@ -1159,17 +1223,19 @@ impl Node { if transfer.state != MediaTransferState::Paused { return Err(NodeError::InvalidAttachment); } - if transfer.direction == MediaDirection::Inbound { - for object in self.store.media_objects_for_transfer(transfer_id)? { - if object.state != MediaTransferState::Complete { - self.store.set_media_object_state( - &object.local_id, - MediaTransferState::Queued, - rng, - )?; - } - } - } + let object_pairs = if transfer.direction == MediaDirection::Inbound { + self.store + .media_objects_for_transfer(transfer_id)? + .into_iter() + .filter(|object| object.state != MediaTransferState::Complete) + .map(|object| { + let after = media_object_with_state(&object, MediaTransferState::Queued); + (object, after) + }) + .collect::>() + } else { + Vec::new() + }; let state = if transfer.direction == MediaDirection::Outbound && self.attachment_manifest_was_queued(&transfer)? { @@ -1177,8 +1243,10 @@ impl Node { } else { MediaTransferState::Queued }; - self.store - .set_media_transfer_state(transfer_id, state, now, rng)?; + let mut transfer_after = transfer.clone(); + transfer_after.state = state; + transfer_after.updated_at = now; + self.commit_attachment_state(&transfer, &transfer_after, &object_pairs, &[], rng)?; self.attachment_request_at.remove(transfer_id); self.attachment_request_target.remove(transfer_id); self.emit_attachment_update(transfer_id) @@ -1230,14 +1298,14 @@ impl Node { Ok((transfer, objects)) } - pub(crate) fn record_group_attachment_offer( - &mut self, + pub(crate) fn prepare_group_attachment_offer( + &self, offer: GroupAttachmentOffer, content_id: [u8; 16], manifest: &AttachmentManifest<'_>, now: u64, rng: &mut impl CryptoRngCore, - ) -> Result<[u8; 16]> { + ) -> Result<(MediaTransferRecord, Vec)> { let mut transfer_id = [0u8; 16]; rng.fill_bytes(&mut transfer_id); let transfer = MediaTransferRecord { @@ -1252,30 +1320,27 @@ impl Node { state: MediaTransferState::AwaitingConsent, updated_at: now, }; - self.store.put_media_transfer(&transfer, rng)?; + let mut objects = Vec::new(); for descriptor in core::iter::once(&manifest.primary).chain(manifest.preview.as_ref()) { let mut local_id = [0u8; 16]; rng.fill_bytes(&mut local_id); - self.store.put_media_object( - &MediaObjectRecord { - local_id, - transfer_id, - object_id: descriptor.object_id, - role: descriptor.role as u8, - total_len: descriptor.total_len, - chunk_count: descriptor.chunk_count, - content_hash: descriptor.content_hash, - media_type: descriptor.media_type.to_owned(), - filename: descriptor.filename.map(str::to_owned), - state: MediaTransferState::AwaitingConsent, - verified_bitmap: vec![0; (descriptor.chunk_count as usize).div_ceil(8)], - chunk_addresses: vec![None; descriptor.chunk_count as usize], - verified_bytes: 0, - }, - rng, - )?; + objects.push(MediaObjectRecord { + local_id, + transfer_id, + object_id: descriptor.object_id, + role: descriptor.role as u8, + total_len: descriptor.total_len, + chunk_count: descriptor.chunk_count, + content_hash: descriptor.content_hash, + media_type: descriptor.media_type.to_owned(), + filename: descriptor.filename.map(str::to_owned), + state: MediaTransferState::AwaitingConsent, + verified_bitmap: vec![0; (descriptor.chunk_count as usize).div_ceil(8)], + chunk_addresses: vec![None; descriptor.chunk_count as usize], + verified_bytes: 0, + }); } - Ok(transfer_id) + Ok((transfer, objects)) } pub(crate) async fn activate_attachment_transfers( @@ -1308,15 +1373,14 @@ impl Node { match transfer.direction { MediaDirection::Outbound if transfer.scope == MediaScope::Pairwise - && transfer.state == MediaTransferState::Queued => + && transfer.state == MediaTransferState::Queued + && self + .store + .media_objects_for_transfer(&transfer.local_id)? + .iter() + .all(|object| object.state == MediaTransferState::Complete) => { if self.queue_pairwise_attachment_manifest(transfer, now, rng)? { - self.store.set_media_transfer_state( - &transfer.local_id, - MediaTransferState::Transferring, - now, - rng, - )?; self.emit_attachment_update(&transfer.local_id)?; } } @@ -1330,7 +1394,7 @@ impl Node { now.saturating_sub(*last) >= MISSING_RETRY_SECS })) => { - let mut queued = false; + let mut padded = Vec::new(); let verified_before = self.verified_attachment_chunk_count(&transfer.local_id)?; let mut requested_chunks = 0usize; @@ -1345,13 +1409,7 @@ impl Node { content_hash: object.content_hash, }, )?; - self.queue_attachment_bulk_to_account( - &transfer.peer, - &complete, - now, - rng, - )?; - queued = true; + padded.extend(Self::encode_attachment_bulk_records(&[&complete])?); continue; } let ranges = missing_ranges(&object); @@ -1367,23 +1425,32 @@ impl Node { object.object_id, AttachmentBulkOperation::RequestMissing { role, ranges }, )?; - self.queue_attachment_bulk_to_account(&transfer.peer, &record, now, rng)?; - queued = true; + padded.extend(Self::encode_attachment_bulk_records(&[&record])?); } - if queued { - self.store.set_media_transfer_state( - &transfer.local_id, - if self - .store - .media_objects_for_transfer(&transfer.local_id)? - .iter() - .all(|object| object.state == MediaTransferState::Complete) - { - MediaTransferState::Complete - } else { - MediaTransferState::Transferring - }, - transfer.updated_at, + if !padded.is_empty() { + let mut transfer_after = transfer.clone(); + transfer_after.state = if self + .store + .media_objects_for_transfer(&transfer.local_id)? + .iter() + .all(|object| object.state == MediaTransferState::Complete) + { + MediaTransferState::Complete + } else { + MediaTransferState::Transferring + }; + let transfer_transition = + (transfer != &transfer_after).then_some(MediaTransferTransition { + before: transfer, + after: &transfer_after, + }); + self.queue_attachment_padded_to_account( + &transfer.peer, + &padded, + transfer_transition.as_slice(), + &[], + &[], + now, rng, )?; if requested_chunks == 0 { @@ -1424,9 +1491,21 @@ impl Node { _ => unreachable!("terminal state checked above"), }; let terminal = self.bulk_record(transfer, object.object_id, operation)?; - self.queue_attachment_bulk_to_account(&transfer.peer, &terminal, now, rng)?; - self.store - .set_media_transfer_state(&transfer.local_id, transfer.state, 0, rng)?; + let mut transfer_after = transfer.clone(); + transfer_after.updated_at = 0; + let transition = MediaTransferTransition { + before: transfer, + after: &transfer_after, + }; + self.queue_attachment_bulk_records_to_account( + &transfer.peer, + &[&terminal], + &[transition], + &[], + &[], + now, + rng, + )?; } } @@ -1454,21 +1533,18 @@ impl Node { if transfer.state != MediaTransferState::Queued || !self.peer_supports_attachment(&transfer.peer)? || !self.carrier_allows_bulk(&transfer.peer, now)? + || !self + .store + .media_objects_for_transfer(&transfer.local_id)? + .iter() + .all(|object| object.state == MediaTransferState::Complete) { eligible = false; break; } } - if eligible && self.queue_group_attachment_manifest(&group, &content_id, now, rng)? { - for transfer in copies { - self.store.set_media_transfer_state( - &transfer.local_id, - MediaTransferState::Transferring, - now, - rng, - )?; - self.emit_attachment_update(&transfer.local_id)?; - } + if eligible { + self.queue_group_attachment_manifest(&group, &content_id, now, rng)?; } } Ok(()) @@ -1480,11 +1556,12 @@ impl Node { peer_device: [u8; 32], body: &[u8], now: u64, + control: &DeferredControlRecord, rng: &mut impl CryptoRngCore, - ) -> Result<()> { + ) -> Result { let DecodedAttachmentBulkRecord::Record(record) = decode_attachment_bulk_record(body) else { - return Ok(()); + return Ok(false); }; match record.operation { AttachmentBulkOperation::RequestMissing { role, ref ranges } => { @@ -1494,45 +1571,68 @@ impl Node { MediaDirection::Outbound, ))? else { - return Ok(()); + return Ok(false); }; if transfer.state == MediaTransferState::Paused { - return Ok(()); + return Ok(false); } let Some(object) = ignore_unknown(self.resolve_bulk_object(&transfer, record.object_id, role))? else { - return Ok(()); + return Ok(false); }; if validate_missing_ranges(ranges, object.chunk_count).is_err() { - return Ok(()); + return Ok(false); } let mut served = 0usize; + let mut chunks = Vec::new(); 'ranges: for range in ranges { for index in range.start..range.start + range.count { if served == MAX_CHUNKS_PER_REQUEST { break 'ranges; } let sealed = self.store.read_media_chunk(&object.local_id, index)?; - let chunk = self.bulk_record( + chunks.push((index, sealed)); + served += 1; + } + } + if chunks.is_empty() { + return Ok(false); + } + let records = chunks + .iter() + .map(|(index, sealed)| { + self.bulk_record( &transfer, object.object_id, AttachmentBulkOperation::Chunk { role, - index, - sealed_chunk: &sealed, + index: *index, + sealed_chunk: sealed, }, - )?; - self.queue_attachment_bulk(&peer_device, &chunk, now, rng)?; - served += 1; - } - } - self.store.set_media_transfer_state( - &transfer.local_id, - MediaTransferState::Transferring, + ) + }) + .collect::>>()?; + let record_refs = records.iter().collect::>(); + let mut transfer_after = transfer.clone(); + transfer_after.state = MediaTransferState::Transferring; + transfer_after.updated_at = now; + let transfer_transition = + (transfer != transfer_after).then_some(MediaTransferTransition { + before: &transfer, + after: &transfer_after, + }); + self.queue_attachment_bulk_records( + &[peer_device], + &record_refs, + transfer_transition.as_slice(), + &[], + core::slice::from_ref(control), now, rng, )?; + self.emit_attachment_update(&transfer.local_id)?; + Ok(true) } AttachmentBulkOperation::Chunk { role, @@ -1545,124 +1645,95 @@ impl Node { MediaDirection::Inbound, ))? else { - return Ok(()); + return Ok(false); }; let Some(object) = ignore_unknown(self.resolve_bulk_object(&transfer, record.object_id, role))? else { - return Ok(()); + return Ok(false); }; if index >= object.chunk_count { - return Ok(()); + return Ok(false); } let manifest = self.load_manifest(&transfer)?; let Some(manifest_object) = manifest.objects.iter().find(|candidate| { candidate.object_id == object.object_id && candidate.role == role }) else { - return Ok(()); + return Ok(false); }; let context = self.chunk_context(&transfer, manifest_object); if open_attachment_chunk(&manifest.attachment_key, &context, index, sealed_chunk) .is_err() { - let corrupt = self.bulk_record( + self.finish_attachment_from_control( &transfer, object.object_id, - AttachmentBulkOperation::Cancel(AttachmentReason::Corrupt), - )?; - self.finish_attachment_locally( - &transfer, MediaTransferState::Corrupt, + Some(AttachmentBulkOperation::Cancel(AttachmentReason::Corrupt)), + peer_device, + control, now, - false, rng, )?; - self.queue_attachment_bulk(&peer_device, &corrupt, now, rng)?; - return Ok(()); - } - match self - .store - .commit_media_chunk(&object.local_id, index, sealed_chunk, rng) - { - Ok(_) => {} - Err(StoreError::MediaQuota) => { - let reject = self.bulk_record( - &transfer, - object.object_id, - AttachmentBulkOperation::Reject(AttachmentReason::Quota), - )?; - self.finish_attachment_locally( - &transfer, - MediaTransferState::Rejected, - now, - false, - rng, - )?; - self.queue_attachment_bulk(&peer_device, &reject, now, rng)?; - return Ok(()); - } - Err(StoreError::LowStorage) => { - let reject = self.bulk_record( - &transfer, - object.object_id, - AttachmentBulkOperation::Reject(AttachmentReason::LowStorage), - )?; - self.finish_attachment_locally( - &transfer, - MediaTransferState::Rejected, - now, - false, - rng, - )?; - self.queue_attachment_bulk(&peer_device, &reject, now, rng)?; - return Ok(()); - } - Err(StoreError::MediaState) => { - let corrupt = self.bulk_record( - &transfer, - object.object_id, - AttachmentBulkOperation::Cancel(AttachmentReason::Corrupt), - )?; - self.finish_attachment_locally( - &transfer, - MediaTransferState::Corrupt, - now, - false, - rng, - )?; - self.queue_attachment_bulk(&peer_device, &corrupt, now, rng)?; - return Ok(()); - } - Err(error) => return Err(error.into()), + return Ok(true); } - self.store.set_media_transfer_state( - &transfer.local_id, - MediaTransferState::Transferring, - now, - rng, - )?; - let stored = match self.store.get_media_object(&object.local_id)? { - Some(MediaRecord::Available(stored)) => stored, - _ => return Err(NodeError::UnknownAttachment), - }; - if let Some(target) = self - .attachment_request_target - .get(&transfer.local_id) - .copied() - { - let verified = self.verified_attachment_chunk_count(&transfer.local_id)?; - if verified >= target { - // The bounded window made progress. Let this same - // receive tick request the next window instead of - // treating the 30-second loss-retry timer as pacing. - self.attachment_request_at.remove(&transfer.local_id); - self.attachment_request_target.remove(&transfer.local_id); - } - } - if stored.chunk_addresses.iter().all(Option::is_some) { + let staged = + match self + .store + .stage_media_chunk(&object.local_id, index, sealed_chunk, rng) + { + Ok(staged) => staged, + Err(StoreError::MediaQuota) => { + self.finish_attachment_from_control( + &transfer, + object.object_id, + MediaTransferState::Rejected, + Some(AttachmentBulkOperation::Reject(AttachmentReason::Quota)), + peer_device, + control, + now, + rng, + )?; + return Ok(true); + } + Err(StoreError::LowStorage) => { + self.finish_attachment_from_control( + &transfer, + object.object_id, + MediaTransferState::Rejected, + Some(AttachmentBulkOperation::Reject( + AttachmentReason::LowStorage, + )), + peer_device, + control, + now, + rng, + )?; + return Ok(true); + } + Err(StoreError::MediaState) => { + self.finish_attachment_from_control( + &transfer, + object.object_id, + MediaTransferState::Corrupt, + Some(AttachmentBulkOperation::Cancel(AttachmentReason::Corrupt)), + peer_device, + control, + now, + rng, + )?; + return Ok(true); + } + Err(error) => return Err(error.into()), + }; + let mut object_after = staged.after; + let mut complete_response = None; + if object_after.chunk_addresses.iter().all(Option::is_some) { let mut hasher = blake3::Hasher::new(); - for chunk_index in 0..stored.chunk_count { - let sealed = self.store.read_media_chunk(&stored.local_id, chunk_index)?; + for chunk_index in 0..object_after.chunk_count { + let sealed = self + .store + .read_media_chunk_from_record(&object_after, chunk_index)?; let plain = Zeroizing::new(open_attachment_chunk( &manifest.attachment_key, &context, @@ -1672,48 +1743,85 @@ impl Node { hasher.update(&plain); } let verified_hash = *hasher.finalize().as_bytes(); - if verified_hash != stored.content_hash { - let corrupt = self.bulk_record( - &transfer, - stored.object_id, - AttachmentBulkOperation::Cancel(AttachmentReason::Corrupt), - )?; - self.finish_attachment_locally( + if verified_hash != object_after.content_hash { + self.finish_attachment_from_control( &transfer, + object_after.object_id, MediaTransferState::Corrupt, + Some(AttachmentBulkOperation::Cancel(AttachmentReason::Corrupt)), + peer_device, + control, now, - false, rng, )?; - self.queue_attachment_bulk(&peer_device, &corrupt, now, rng)?; - return Ok(()); + return Ok(true); } - self.store - .mark_media_complete(&stored.local_id, &verified_hash, rng)?; - let complete = self.bulk_record( - &transfer, - stored.object_id, - AttachmentBulkOperation::Complete { - role, - content_hash: verified_hash, - }, - )?; - self.queue_attachment_bulk(&peer_device, &complete, now, rng)?; - let all_complete = self + object_after.state = MediaTransferState::Complete; + complete_response = Some(AttachmentBulkOperation::Complete { + role, + content_hash: verified_hash, + }); + } + let all_complete = object_after.state == MediaTransferState::Complete + && self .store .media_objects_for_transfer(&transfer.local_id)? .iter() + .filter(|candidate| candidate.local_id != object_after.local_id) .all(|candidate| candidate.state == MediaTransferState::Complete); - if all_complete { - self.store.set_media_transfer_state( - &transfer.local_id, - MediaTransferState::Complete, - now, - rng, - )?; + let mut transfer_after = transfer.clone(); + transfer_after.state = if all_complete { + MediaTransferState::Complete + } else { + MediaTransferState::Transferring + }; + transfer_after.updated_at = now; + let object_pairs = (staged.before != object_after) + .then_some((staged.before, object_after)) + .into_iter() + .collect::>(); + if let Some(operation) = complete_response { + let complete = self.bulk_record(&transfer, record.object_id, operation)?; + let transfer_transition = + (transfer != transfer_after).then_some(MediaTransferTransition { + before: &transfer, + after: &transfer_after, + }); + let object_transitions = object_pairs + .iter() + .map(|(before, after)| MediaObjectTransition { before, after }) + .collect::>(); + self.queue_attachment_bulk_records( + &[peer_device], + &[&complete], + transfer_transition.as_slice(), + &object_transitions, + core::slice::from_ref(control), + now, + rng, + )?; + } else { + self.commit_attachment_state( + &transfer, + &transfer_after, + &object_pairs, + core::slice::from_ref(control), + rng, + )?; + } + if let Some(target) = self + .attachment_request_target + .get(&transfer.local_id) + .copied() + { + let verified = self.verified_attachment_chunk_count(&transfer.local_id)?; + if verified >= target { + self.attachment_request_at.remove(&transfer.local_id); + self.attachment_request_target.remove(&transfer.local_id); } } self.emit_attachment_update(&transfer.local_id)?; + Ok(true) } AttachmentBulkOperation::Complete { role, content_hash } => { let Some(transfer) = ignore_unknown(self.resolve_bulk_transfer( @@ -1722,15 +1830,15 @@ impl Node { MediaDirection::Outbound, ))? else { - return Ok(()); + return Ok(false); }; let Some(object) = ignore_unknown(self.resolve_bulk_object(&transfer, record.object_id, role))? else { - return Ok(()); + return Ok(false); }; if object.content_hash != content_hash { - return Ok(()); + return Ok(false); } // Completion acknowledgements can cross an explicit local // terminal decision in flight. A delayed acknowledgement @@ -1742,21 +1850,26 @@ impl Node { | MediaTransferState::Cancelled | MediaTransferState::Corrupt ) { - return Ok(()); + return Ok(false); } - self.store.set_media_transfer_state( - &transfer.local_id, - MediaTransferState::Complete, - now, + let mut transfer_after = transfer.clone(); + transfer_after.state = MediaTransferState::Complete; + transfer_after.updated_at = now; + self.commit_attachment_state( + &transfer, + &transfer_after, + &[], + core::slice::from_ref(control), rng, )?; self.emit_attachment_update(&transfer.local_id)?; + Ok(true) } AttachmentBulkOperation::Cancel(_) => { let Some(transfer) = ignore_unknown(self.resolve_bulk_transfer_any_direction(&record, &peer))? else { - return Ok(()); + return Ok(false); }; if !self .store @@ -1764,15 +1877,19 @@ impl Node { .iter() .any(|object| object.object_id == record.object_id) { - return Ok(()); + return Ok(false); } - self.finish_attachment_locally( + self.finish_attachment_from_control( &transfer, + record.object_id, MediaTransferState::Cancelled, + None, + peer_device, + control, now, - false, rng, )?; + Ok(true) } AttachmentBulkOperation::Reject(_) => { let Some(transfer) = ignore_unknown(self.resolve_bulk_transfer( @@ -1781,7 +1898,7 @@ impl Node { MediaDirection::Outbound, ))? else { - return Ok(()); + return Ok(false); }; if !self .store @@ -1789,18 +1906,21 @@ impl Node { .iter() .any(|object| object.object_id == record.object_id) { - return Ok(()); + return Ok(false); } - self.finish_attachment_locally( + self.finish_attachment_from_control( &transfer, + record.object_id, MediaTransferState::Rejected, + None, + peer_device, + control, now, - false, rng, )?; + Ok(true) } } - Ok(()) } pub(crate) fn peer_supports_attachment(&self, peer: &[u8; 32]) -> Result { @@ -1921,7 +2041,14 @@ impl Node { prepared.push((route, before, after)); } if prepared.is_empty() { - return Ok(message_before.wire_id.is_some()); + if message_before.wire_id.is_none() { + return Ok(false); + } + let mut transfer_after = transfer.clone(); + transfer_after.state = MediaTransferState::Transferring; + transfer_after.updated_at = now; + self.commit_attachment_state(transfer, &transfer_after, &[], &[], rng)?; + return Ok(true); } let session_transitions = prepared .iter() @@ -1935,6 +2062,13 @@ impl Node { .iter() .map(|(before, after)| kult_store::DeliveryTransition { before, after }) .collect::>(); + let mut transfer_after = transfer.clone(); + transfer_after.state = MediaTransferState::Transferring; + transfer_after.updated_at = now; + let media_transfers = [MediaTransferTransition { + before: transfer, + after: &transfer_after, + }]; self.store.commit_plan( CommitPlan::PairwiseSend(PairwiseSendPlan { sessions: &session_transitions, @@ -1946,10 +2080,15 @@ impl Node { deliveries: &new_deliveries, delivery_updates: &delivery_updates, queue: &queue, + groups: &[], + authorities: &[], scheduled: None, clear_capabilities: &[], clear_reset_markers: &[], ephemeral: None, + media_transfers: &media_transfers, + media_objects: &[], + delete_controls: &[], presentation_changed: false, }), rng, @@ -1962,44 +2101,122 @@ impl Node { Ok(true) } - fn queue_attachment_bulk_to_account( + #[allow(clippy::too_many_arguments)] // exact control and durable consequence boundary + fn queue_attachment_bulk_records_to_account( &mut self, account: &[u8; 32], - record: &AttachmentBulkRecord<'_>, + records: &[&AttachmentBulkRecord<'_>], + media_transfers: &[MediaTransferTransition<'_>], + media_objects: &[MediaObjectTransition<'_>], + delete_controls: &[DeferredControlRecord], + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result<()> { + let padded = Self::encode_attachment_bulk_records(records)?; + self.queue_attachment_padded_to_account( + account, + &padded, + media_transfers, + media_objects, + delete_controls, + now, + rng, + ) + } + + #[allow(clippy::too_many_arguments)] // exact control and durable consequence boundary + fn queue_attachment_padded_to_account( + &mut self, + account: &[u8; 32], + padded: &[Vec], + media_transfers: &[MediaTransferTransition<'_>], + media_objects: &[MediaObjectTransition<'_>], + delete_controls: &[DeferredControlRecord], now: u64, rng: &mut impl CryptoRngCore, ) -> Result<()> { let endpoints = self.store.contact_devices_for(account)?; - if endpoints.is_empty() { - return self.queue_attachment_bulk(account, record, now, rng); - } - if endpoints + let devices = if endpoints.is_empty() { + vec![*account] + } else { + endpoints + .iter() + .map(|endpoint| endpoint.device) + .collect::>() + }; + if devices .iter() - .any(|endpoint| !self.sessions.contains_key(&endpoint.device)) + .any(|device| !self.sessions.contains_key(device)) { return Err(NodeError::NoSession); } - for endpoint in endpoints { - self.queue_attachment_bulk(&endpoint.device, record, now, rng)?; - } - Ok(()) + self.queue_attachment_padded( + &devices, + padded, + media_transfers, + media_objects, + delete_controls, + now, + rng, + ) } - fn queue_attachment_bulk( + #[allow(clippy::too_many_arguments)] // exact control and durable consequence boundary + fn queue_attachment_bulk_records( &mut self, - peer: &[u8; 32], - record: &AttachmentBulkRecord<'_>, + devices: &[[u8; 32]], + records: &[&AttachmentBulkRecord<'_>], + media_transfers: &[MediaTransferTransition<'_>], + media_objects: &[MediaObjectTransition<'_>], + delete_controls: &[DeferredControlRecord], now: u64, rng: &mut impl CryptoRngCore, ) -> Result<()> { - let payload = encode_attachment_bulk_record(record)?; - let padded = pad_to_minimum(&payload, BULK_CONTROL_PADDING_FLOOR)?; - self.commit_pairwise_envelopes( - &[*peer], + let padded = Self::encode_attachment_bulk_records(records)?; + self.queue_attachment_padded( + devices, &padded, + media_transfers, + media_objects, + delete_controls, + now, + rng, + ) + } + + fn encode_attachment_bulk_records( + records: &[&AttachmentBulkRecord<'_>], + ) -> Result>> { + records + .iter() + .map(|record| -> Result> { + let payload = encode_attachment_bulk_record(record)?; + Ok(pad_to_minimum(&payload, BULK_CONTROL_PADDING_FLOOR)?) + }) + .collect() + } + + #[allow(clippy::too_many_arguments)] // exact control and durable consequence boundary + fn queue_attachment_padded( + &mut self, + devices: &[[u8; 32]], + padded: &[Vec], + media_transfers: &[MediaTransferTransition<'_>], + media_objects: &[MediaObjectTransition<'_>], + delete_controls: &[DeferredControlRecord], + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result<()> { + let payloads = padded.iter().map(Vec::as_slice).collect::>(); + self.commit_pairwise_payloads_with_effects( + devices, + &payloads, EnvelopeKind::Receipt, QueueClass::Bulk, None, + media_transfers, + media_objects, + delete_controls, now, rng, )?; @@ -2083,6 +2300,36 @@ impl Node { .ok_or(NodeError::UnknownAttachment) } + fn commit_attachment_state( + &mut self, + transfer_before: &MediaTransferRecord, + transfer_after: &MediaTransferRecord, + object_pairs: &[(MediaObjectRecord, MediaObjectRecord)], + delete_controls: &[DeferredControlRecord], + rng: &mut impl CryptoRngCore, + ) -> Result<()> { + let transfer = (transfer_before != transfer_after).then_some(MediaTransferTransition { + before: transfer_before, + after: transfer_after, + }); + let objects = object_pairs + .iter() + .filter(|(before, after)| before != after) + .map(|(before, after)| MediaObjectTransition { before, after }) + .collect::>(); + let receipt = self.store.commit_plan( + CommitPlan::AttachmentState(AttachmentStatePlan { + media_transfers: transfer.as_slice(), + media_objects: &objects, + delete_controls, + presentation_changed: true, + }), + rng, + )?; + self.accept_commit_receipt(receipt, []); + Ok(()) + } + fn require_attachment(&self, transfer_id: &[u8; 16]) -> Result { match self.store.get_media_transfer(transfer_id)? { Some(MediaRecord::Available(transfer)) => Ok(transfer), @@ -2152,20 +2399,93 @@ impl Node { notify_remote: bool, rng: &mut impl CryptoRngCore, ) -> Result<()> { - if transfer.direction == MediaDirection::Inbound || state == MediaTransferState::Corrupt { - for object in self.store.media_objects_for_transfer(&transfer.local_id)? { - self.store - .set_media_object_state(&object.local_id, state, rng)?; - } + let object_pairs = if transfer.direction == MediaDirection::Inbound + || state == MediaTransferState::Corrupt + { + self.store + .media_objects_for_transfer(&transfer.local_id)? + .into_iter() + .map(|object| { + let after = media_object_with_state(&object, state); + (object, after) + }) + .collect::>() + } else { + Vec::new() + }; + let mut transfer_after = transfer.clone(); + transfer_after.state = state; + transfer_after.updated_at = if notify_remote { now } else { 0 }; + self.commit_attachment_state(transfer, &transfer_after, &object_pairs, &[], rng)?; + self.attachment_request_at.remove(&transfer.local_id); + self.attachment_request_target.remove(&transfer.local_id); + self.emit_attachment_update(&transfer.local_id) + } + + #[allow(clippy::too_many_arguments)] // authenticated control, response, and state boundary + fn finish_attachment_from_control( + &mut self, + transfer: &MediaTransferRecord, + object_id: [u8; 16], + state: MediaTransferState, + response: Option>, + peer_device: [u8; 32], + control: &DeferredControlRecord, + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result<()> { + let object_pairs = if transfer.direction == MediaDirection::Inbound + || state == MediaTransferState::Corrupt + { + self.store + .media_objects_for_transfer(&transfer.local_id)? + .into_iter() + .map(|object| { + let after = media_object_with_state(&object, state); + (object, after) + }) + .collect::>() + } else { + Vec::new() + }; + let mut transfer_after = transfer.clone(); + transfer_after.state = state; + transfer_after.updated_at = 0; + if let Some(operation) = response { + let record = self.bulk_record(transfer, object_id, operation)?; + let transfer_transition = + (transfer != &transfer_after).then_some(MediaTransferTransition { + before: transfer, + after: &transfer_after, + }); + let object_transitions = object_pairs + .iter() + .filter(|(before, after)| before != after) + .map(|(before, after)| MediaObjectTransition { before, after }) + .collect::>(); + self.queue_attachment_bulk_records( + &[peer_device], + &[&record], + transfer_transition.as_slice(), + &object_transitions, + core::slice::from_ref(control), + now, + rng, + )?; + } else { + self.commit_attachment_state( + transfer, + &transfer_after, + &object_pairs, + core::slice::from_ref(control), + rng, + )?; } - self.store.set_media_transfer_state( - &transfer.local_id, - state, - if notify_remote { now } else { 0 }, - rng, - )?; self.attachment_request_at.remove(&transfer.local_id); self.attachment_request_target.remove(&transfer.local_id); + if !object_pairs.is_empty() { + self.store.collect_media_garbage()?; + } self.emit_attachment_update(&transfer.local_id) } @@ -2359,8 +2679,24 @@ mod tests { operation: AttachmentBulkOperation::Cancel(AttachmentReason::User), }; let encoded = encode_attachment_bulk_record(&record).unwrap(); - node.apply_attachment_bulk([2; 32], [2; 32], &encoded, 1_800_000_000, &mut rng) - .unwrap(); + let control = DeferredControlRecord { + content_id: [5; 16], + peer: [2; 32], + peer_device: [2; 32], + kind: kult_store::DeferredControlKind::AttachmentBulk, + body: encoded.clone(), + received_at: 1_800_000_000, + }; + assert!(!node + .apply_attachment_bulk( + [2; 32], + [2; 32], + &encoded, + 1_800_000_000, + &control, + &mut rng, + ) + .unwrap()); assert!(node.attachments().unwrap().is_empty()); assert!(node.drain_events().is_empty()); } diff --git a/crates/kult-node/src/authority.rs b/crates/kult-node/src/authority.rs index ac8786d..e8f77de 100644 --- a/crates/kult-node/src/authority.rs +++ b/crates/kult-node/src/authority.rs @@ -5,7 +5,7 @@ use sha2::{Digest, Sha256}; use kult_crypto::{ verify_group_admin_request_signature, verify_group_authority_state_signature, - verify_group_owner_transfer_signature, GroupReceiverChain, GroupSenderChain, IdentityPublic, + verify_group_owner_transfer_signature, GroupSenderChain, IdentityPublic, }; use kult_protocol::{ decode_group_authority, encode_group_authority, encode_group_authority_state, @@ -15,9 +15,12 @@ use kult_protocol::{ GroupMemberInfo, GroupRole, OwnerTransferCertificate, SignedGroupAuthorityState, CONTENT_KIND_GROUP_AUTHORITY, MAX_GROUP_ADMIN_REQUESTS, MAX_GROUP_NAME_LEN, }; -use kult_store::{GroupAuthorityRecord, GroupMember, GroupRecord}; +use kult_store::{ + ContactTransition, DeferredControlRecord, GroupAuthorityRecord, GroupAuthorityStateTransition, + GroupChainStateTransition, GroupMember, GroupRecord, GroupStatePlan, GroupStateTransition, +}; -use crate::{Event, GroupAuthorityInfo, GroupMemberRoleInfo, Node, NodeError, Result}; +use crate::{CommitPlan, Event, GroupAuthorityInfo, GroupMemberRoleInfo, Node, NodeError, Result}; const ID_RETRY_LIMIT: usize = 16; @@ -277,7 +280,14 @@ impl Node { state.secret_hash = hash_secret(&secret); // The transfer state itself is still authorized by the old owner. state.signature = [0; 64]; - self.commit_authority_state_signed_by(state, secret, now, rng, self.identity.public().ed) + self.commit_authority_state_signed_by( + state, + secret, + now, + rng, + self.identity.public().ed, + None, + ) } pub(crate) fn group_authority_add_member( @@ -500,7 +510,19 @@ impl Node { rng: &mut impl CryptoRngCore, ) -> Result<[u8; 16]> { let signer = state.signer; - self.commit_authority_state_signed_by(state, secret, now, rng, signer) + self.commit_authority_state_signed_by(state, secret, now, rng, signer, None) + } + + fn commit_authority_state_for_request( + &mut self, + state: SignedGroupAuthorityState, + secret: [u8; 32], + request_id: [u8; 16], + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result<[u8; 16]> { + let signer = state.signer; + self.commit_authority_state_signed_by(state, secret, now, rng, signer, Some(request_id)) } fn commit_authority_state_signed_by( @@ -510,6 +532,7 @@ impl Node { now: u64, rng: &mut impl CryptoRngCore, signer: [u8; 32], + consumed_request: Option<[u8; 16]>, ) -> Result<[u8; 16]> { if signer != self.identity.public().ed { return Err(NodeError::NotGroupOwner); @@ -523,45 +546,43 @@ impl Node { encode_group_authority_state(&state).map_err(|_| NodeError::InvalidGroupAuthority)?; let wire = encode_group_authority(id, &payload).map_err(|_| NodeError::InvalidGroupAuthority)?; - self.group_send_content_with_id(&state.group, wire, id, now, now, rng)?; - - // Reload because sending advanced the old sender chain. - let mut rec = self - .store - .get_group(&state.group)? - .ok_or(NodeError::UnknownGroup)?; - rec.prev_secret = Some(rec.secret); - rec.secret = secret; - rec.name = state.name.clone(); - rec.creator = state.owner; - rec.generation = state.generation; - rec.members = state - .members - .iter() - .map(|member| GroupMember { - peer: member.peer, - identity: member.identity.clone(), - }) - .collect(); - self.rotate_group(&mut rec, rng)?; - self.store.put_group(&rec, rng)?; - let consumed_requests = self - .store - .get_group_authority(&state.group)? - .map(|record| record.consumed_requests) + let authority_before = self.store.get_group_authority(&state.group)?; + let consumed_requests = authority_before + .as_ref() + .map(|record| record.consumed_requests.clone()) .unwrap_or_default(); - self.store.put_group_authority( - &GroupAuthorityRecord { + let mut consumed_requests = consumed_requests; + if let Some(request_id) = consumed_request { + if !consumed_requests.contains(&request_id) { + consumed_requests.push(request_id); + } + if consumed_requests.len() > MAX_GROUP_ADMIN_REQUESTS { + let excess = consumed_requests.len() - MAX_GROUP_ADMIN_REQUESTS; + consumed_requests.drain(..excess); + } + } + let authority = crate::groups::PreparedAuthorityGroupState { + secret, + name: state.name.clone(), + creator: state.owner, + generation: state.generation, + members: state + .members + .iter() + .map(|member| GroupMember { + peer: member.peer, + identity: member.identity.clone(), + }) + .collect(), + authority_before, + authority_after: GroupAuthorityRecord { group: state.group, state_id: id, state_payload: payload, consumed_requests, }, - rng, - )?; - self.events - .push_back(Event::GroupUpdated { group: state.group }); - Ok(id) + }; + self.group_send_authority_content_with_id(&state.group, wire, id, now, now, &authority, rng) } fn mint_authority_id( @@ -583,29 +604,31 @@ impl Node { pub(crate) fn apply_authority_announce( &mut self, peer: [u8; 32], + peer_device: [u8; 32], announce: &GroupAuthorityAnnounce, + control: &DeferredControlRecord, rng: &mut impl CryptoRngCore, established: &mut bool, - ) -> Result { + ) -> Result<(bool, bool)> { let DecodedGroupAuthority::State(state) = decode_group_authority(&announce.state_payload) else { - return Ok(true); + return Ok((true, false)); }; if state.group != announce.group || !state.members.iter().any(|member| member.peer == peer) || verify_authority_state(&state, Some(&announce.secret)).is_err() { - return Ok(true); + return Ok((true, false)); } let me = self.identity.public().ed; if !state.members.iter().any(|member| member.peer == me) { - return Ok(false); + return Ok((false, false)); } let current_authority = self.store.get_group_authority(&state.group)?; let current_state = current_authority.as_ref().map(decode_stored).transpose()?; if let Some(current) = ¤t_state { if current.original_owner != state.original_owner { - return Ok(true); + return Ok((true, false)); } if state.generation > current.generation && !transfer_chain_extends(¤t.transfers, &state.transfers) @@ -613,11 +636,11 @@ impl Node { // Once one same-generation ownership fork wins locally, a // later state rooted in the losing transfer chain cannot use // a larger generation to take the replica back. - return Ok(true); + return Ok((true, false)); } } else if let Some(group) = self.store.get_group(&state.group)? { if group.creator != state.original_owner { - return Ok(true); + return Ok((true, false)); } } let adopt = current_state.as_ref().is_none_or(|current| { @@ -635,11 +658,15 @@ impl Node { if !adopt && !current_winner { // A losing fork or older state must never replace a receiver // chain belonging to the accepted generation. - return Ok(true); + return Ok((true, false)); } - let mut rec = match self.store.get_group(&state.group)? { - Some(rec) => rec, + let before_group = self.store.get_group(&state.group)?; + let mut rec = match before_group.as_ref() { + Some(rec) => rec.clone(), None => { + if !adopt { + return Err(NodeError::CorruptState); + } let chain = GroupSenderChain::generate(rng); GroupRecord { id: state.group, @@ -659,6 +686,9 @@ impl Node { } } }; + let mut contacts = Vec::new(); + let mut authority_after = None; + let mut removed_peers = Vec::new(); if adopt { let stubs = state .members @@ -668,17 +698,13 @@ impl Node { identity: member.identity.clone(), }) .collect::>(); - self.adopt_roster_stubs(&stubs, rng)?; - let old_peers = rec + contacts = self.prepare_roster_stubs(&stubs)?; + removed_peers = rec .members .iter() .map(|member| member.peer) - .collect::>(); - for old in old_peers { - if !state.members.iter().any(|member| member.peer == old) { - self.store.delete_group_chain(&state.group, &old)?; - } - } + .filter(|old| !state.members.iter().any(|member| member.peer == *old)) + .collect(); rec.prev_secret = (rec.secret != announce.secret).then_some(rec.secret); rec.secret = announce.secret; rec.name = state.name.clone(); @@ -686,41 +712,95 @@ impl Node { rec.members = state_members(&state); rec.generation = state.generation; self.rotate_group(&mut rec, rng)?; - self.store.put_group(&rec, rng)?; - self.store.put_group_authority( - &GroupAuthorityRecord { - group: state.group, - state_id: announce.state_id, - state_payload: announce.state_payload.clone(), - consumed_requests: current_authority - .map(|record| record.consumed_requests) - .unwrap_or_default(), - }, - rng, - )?; - self.events - .push_back(Event::GroupUpdated { group: state.group }); - } - if rec.members.iter().any(|member| member.peer == peer) { - let replace = match self.store.get_group_chain(&state.group, &peer)? { - Some(blob) => postcard::from_bytes::(&blob) - .map(|chain| chain.key_id() != announce.key_id) - .unwrap_or(true), - None => true, - }; - if replace { - let chain = GroupReceiverChain::new( - announce.key_id, - &announce.chain_key, - announce.iteration, - ); - let encoded = postcard::to_allocvec(&chain).map_err(|_| NodeError::CorruptState)?; - self.store - .put_group_chain(&state.group, &peer, &encoded, rng)?; - *established = true; - } + authority_after = Some(GroupAuthorityRecord { + group: state.group, + state_id: announce.state_id, + state_payload: announce.state_payload.clone(), + consumed_requests: current_authority + .as_ref() + .map(|record| record.consumed_requests.clone()) + .unwrap_or_default(), + }); + } + if !rec.members.iter().any(|member| member.peer == peer) { + return Ok((false, false)); + } + let (before_chain, after_chain) = self.prepare_group_receiver_chain( + &state.group, + peer, + peer_device, + announce.key_id, + &announce.chain_key, + announce.iteration, + )?; + let removed_chain_rows = self + .store + .group_chains(&state.group)? + .into_iter() + .filter(|(account, _)| removed_peers.contains(account)) + .collect::>(); + let mut chain_transitions = removed_chain_rows + .iter() + .map(|(account, chain)| GroupChainStateTransition { + group: state.group, + peer: *account, + before: Some(chain.as_slice()), + after: None, + }) + .collect::>(); + if let Some(after_chain) = after_chain.as_ref() { + chain_transitions.push(GroupChainStateTransition { + group: state.group, + peer, + before: before_chain.as_ref().map(|chain| chain.as_slice()), + after: Some(after_chain), + }); } - Ok(true) + let group_transitions = adopt + .then_some(GroupStateTransition { + before: before_group.as_ref(), + after: Some(&rec), + }) + .into_iter() + .collect::>(); + let authority_transitions = authority_after + .as_ref() + .map(|after| GroupAuthorityStateTransition { + before: current_authority.as_ref(), + after: Some(after), + }) + .into_iter() + .collect::>(); + let contact_transitions = contacts + .iter() + .map(|contact| ContactTransition { + before: None, + after: contact, + }) + .collect::>(); + let mut events = contacts + .iter() + .map(|contact| Event::ContactAdded { peer: contact.peer }) + .collect::>(); + if adopt { + events.push(Event::GroupUpdated { group: state.group }); + } + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &group_transitions, + chains: &chain_transitions, + contacts: &contact_transitions, + authorities: &authority_transitions, + delete_controls: core::slice::from_ref(control), + presentation_changed: !events.is_empty(), + }), + rng, + )?; + self.accept_commit_receipt(receipt, events); + if after_chain.is_some() { + *established = true; + } + Ok((true, true)) } pub(crate) fn apply_authority_remove( @@ -729,9 +809,11 @@ impl Node { group: &[u8; 32], state_id: &[u8; 16], payload: &[u8], - ) -> Result { + control: &DeferredControlRecord, + rng: &mut impl CryptoRngCore, + ) -> Result<(bool, bool)> { let DecodedGroupAuthority::State(state) = decode_group_authority(payload) else { - return Ok(true); + return Ok((true, false)); }; if state.group != *group || state.signer != peer @@ -741,7 +823,7 @@ impl Node { .any(|member| member.peer == self.identity.public().ed) || verify_authority_state(&state, None).is_err() { - return Ok(true); + return Ok((true, false)); } if let Some(current) = self.store.get_group_authority(group)? { let current_state = decode_stored(¤t)?; @@ -751,122 +833,315 @@ impl Node { || state.generation < current_state.generation || (state.generation == current_state.generation && *state_id >= current.state_id) { - return Ok(true); + return Ok((true, false)); } } - self.store.delete_group(group)?; - self.events.push_back(Event::GroupUpdated { group: *group }); - Ok(true) + let Some(before) = self.store.get_group(group)? else { + return Ok((true, false)); + }; + self.commit_group_removal(&before, control, rng)?; + Ok((true, true)) } pub(crate) fn apply_group_admin_request( &mut self, peer: [u8; 32], request: &GroupAdminRequest, + control: &DeferredControlRecord, now: u64, rng: &mut impl CryptoRngCore, - ) -> Result { - let Some(mut stored) = self.store.get_group_authority(&request.group)? else { - return Ok(true); + ) -> Result<(bool, bool)> { + let Some(stored) = self.store.get_group_authority(&request.group)? else { + return Ok((true, false)); }; if stored.consumed_requests.contains(&request.request_id) { - return Ok(true); + let state = decode_stored(&stored)?; + let (accepted, state_id, reason) = + self.observed_admin_request_outcome(request, &state)?; + return self.finish_admin_request( + peer, request, control, &stored, accepted, state_id, reason, now, rng, + ); } let state = decode_stored(&stored)?; let signature: [u8; 64] = match request.signature.as_slice().try_into() { Ok(signature) => signature, - Err(_) => return Ok(true), + Err(_) => { + return self.finish_admin_request( + peer, request, control, &stored, false, None, 2, now, rng, + ) + } }; let signing = match group_admin_request_signing_bytes(request) { Ok(signing) => signing, - Err(_) => return Ok(true), + Err(_) => { + return self.finish_admin_request( + peer, request, control, &stored, false, None, 2, now, rng, + ) + } }; let is_admin = state .members .iter() .any(|member| member.peer == peer && member.role == GroupRole::Admin); + let moderation_resume = matches!(request.action, GroupAdminAction::ModeratePoll { .. }) + && request + .base_generation + .checked_add(1) + .is_some_and(|generation| generation == state.generation); let authorized = state.owner == self.identity.public().ed - && request.base_generation == state.generation + && (request.base_generation == state.generation || moderation_resume) && is_admin && verify_group_admin_request_signature(&peer, &signing, &signature).is_ok(); - let result = if authorized { - match &request.action { - GroupAdminAction::Rename { name } => { - self.group_rename(&request.group, name, now, rng).map(Some) + if !authorized { + return self + .finish_admin_request(peer, request, control, &stored, false, None, 2, now, rng); + } + let result = match &request.action { + GroupAdminAction::Rename { name } => { + if name.is_empty() || name.len() > MAX_GROUP_NAME_LEN || moderation_resume { + Err(NodeError::InvalidGroupAuthority) + } else { + let mut next = state.clone(); + next.name.clone_from(name); + let mut secret = [0u8; 32]; + rng.fill_bytes(&mut secret); + prepare_next_state(&mut next, stored.state_id, &secret)?; + self.commit_authority_state_for_request( + next, + secret, + request.request_id, + now, + rng, + ) + .map(Some) } - GroupAdminAction::Invite(member) => self - .group_authority_add_member( - &request.group, - GroupMember { - peer: member.peer, - identity: member.identity.clone(), + } + GroupAdminAction::Invite(member) => { + if moderation_resume { + Err(NodeError::InvalidGroupAuthority) + } else if state + .members + .iter() + .any(|existing| existing.peer == member.peer) + { + Ok(Some(stored.state_id)) + } else { + let mut next = state.clone(); + next.members.push(GroupAuthorityMember { + peer: member.peer, + identity: member.identity.clone(), + role: GroupRole::Member, + }); + next.members.sort_unstable_by_key(|member| member.peer); + let mut secret = [0u8; 32]; + rng.fill_bytes(&mut secret); + prepare_next_state(&mut next, stored.state_id, &secret)?; + self.commit_authority_state_for_request( + next, + secret, + request.request_id, + now, + rng, + ) + .map(Some) + } + } + GroupAdminAction::Remove { peer: removed } => { + if moderation_resume || *removed == state.owner { + Err(NodeError::InvalidGroupAuthority) + } else if !state.members.iter().any(|member| member.peer == *removed) { + Err(NodeError::UnknownPeer) + } else { + let mut next = state.clone(); + next.members.retain(|member| member.peer != *removed); + let mut secret = [0u8; 32]; + rng.fill_bytes(&mut secret); + prepare_next_state(&mut next, stored.state_id, &secret)?; + let state_id = self.commit_authority_state_for_request( + next, + secret, + request.request_id, + now, + rng, + )?; + let authority = self + .store + .get_group_authority(&request.group)? + .ok_or(NodeError::CorruptState)?; + self.queue_group_control( + removed, + &GroupControlPayload::AuthorityRemove { + group: request.group, + state_id, + state_payload: authority.state_payload, }, now, rng, + )?; + Ok(Some(state_id)) + } + } + GroupAdminAction::ModeratePoll { + poll_author, + poll_id, + } => { + let poll = self + .group_polls(&request.group)? + .into_iter() + .find(|poll| poll.author == *poll_author && poll.id == *poll_id) + .ok_or(NodeError::InvalidPoll)?; + if poll.closed { + Ok(None) + } else { + if !moderation_resume { + self.advance_authority(&request.group, now, rng)?; + } + self.commit_group_poll_moderated_close( + &request.group, + *poll_author, + *poll_id, + now, + rng, ) - .map(Some), - GroupAdminAction::Remove { peer } => self - .group_authority_remove_member(&request.group, *peer, now, rng) - .map(Some), - GroupAdminAction::ModeratePoll { - poll_author, - poll_id, - } => self - .group_moderate_poll_close(&request.group, *poll_author, *poll_id, now, rng) - .map(Some), + .map(Some) + } } - } else { - Err(NodeError::InvalidGroupAuthority) }; - stored = self + let current = self .store .get_group_authority(&request.group)? - .unwrap_or(stored); - stored.consumed_requests.push(request.request_id); - if stored.consumed_requests.len() > MAX_GROUP_ADMIN_REQUESTS { - let excess = stored.consumed_requests.len() - MAX_GROUP_ADMIN_REQUESTS; - stored.consumed_requests.drain(..excess); - } - self.store.put_group_authority(&stored, rng)?; + .ok_or(NodeError::CorruptState)?; let (accepted, state_id, reason) = match result { - Ok(state_id) => (true, state_id, 0), + Ok(state_id) => (true, state_id.or(Some(current.state_id)), 0), Err(NodeError::InvalidGroupAuthority) => (false, None, 2), Err(_) => (false, None, 3), }; + self.finish_admin_request( + peer, request, control, ¤t, accepted, state_id, reason, now, rng, + ) + } + + #[allow(clippy::too_many_arguments)] // authenticated request outcome and exact response commit + fn finish_admin_request( + &mut self, + peer: [u8; 32], + request: &GroupAdminRequest, + control: &DeferredControlRecord, + authority_before: &GroupAuthorityRecord, + accepted: bool, + state_id: Option<[u8; 16]>, + reason: u8, + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result<(bool, bool)> { + let mut authority_after = authority_before.clone(); + if !authority_after + .consumed_requests + .contains(&request.request_id) + { + authority_after.consumed_requests.push(request.request_id); + if authority_after.consumed_requests.len() > MAX_GROUP_ADMIN_REQUESTS { + let excess = authority_after.consumed_requests.len() - MAX_GROUP_ADMIN_REQUESTS; + authority_after.consumed_requests.drain(..excess); + } + } + let generation = decode_stored(&authority_after)?.generation; let response = GroupControlPayload::AdminResult(GroupAdminResult { group: request.group, request_id: request.request_id, accepted, - generation: self.group_authority(&request.group)?.generation, + generation, state_id, reason, }); - self.queue_group_control(&peer, &response, now, rng)?; - Ok(true) + let committed = if authority_after == *authority_before { + self.queue_group_control_with_control_ack(&peer, &response, control, now, rng)? + } else { + self.queue_group_control_with_authority_response( + &peer, + &response, + authority_before, + &authority_after, + control, + now, + rng, + )? + }; + Ok((committed, committed)) + } + + fn observed_admin_request_outcome( + &self, + request: &GroupAdminRequest, + state: &SignedGroupAuthorityState, + ) -> Result<(bool, Option<[u8; 16]>, u8)> { + let accepted = match &request.action { + GroupAdminAction::Rename { name } => { + state.generation > request.base_generation && state.name == *name + } + GroupAdminAction::Invite(member) => state + .members + .iter() + .any(|existing| existing.peer == member.peer), + GroupAdminAction::Remove { peer } => { + !state.members.iter().any(|member| member.peer == *peer) + } + GroupAdminAction::ModeratePoll { + poll_author, + poll_id, + } => self + .group_polls(&request.group)? + .into_iter() + .any(|poll| poll.author == *poll_author && poll.id == *poll_id && poll.closed), + }; + let state_id = if accepted { + self.store + .get_group_authority(&request.group)? + .map(|record| record.state_id) + } else { + None + }; + Ok((accepted, state_id, if accepted { 0 } else { 4 })) } pub(crate) fn apply_group_admin_result( &mut self, peer: [u8; 32], result: &GroupAdminResult, - ) -> Result { + control: &DeferredControlRecord, + rng: &mut impl CryptoRngCore, + ) -> Result<(bool, bool)> { let info = self.group_authority(&result.group)?; // The pairwise result and signed authority announce use independent // queued envelopes, so the result may arrive first. Authenticate it // against the currently accepted owner, but do not discard honest UI // status merely because the durable signed state is still in flight. if peer != info.owner { - return Ok(true); - } - self.events.push_back(Event::GroupAdminRequestResolved { - group: result.group, - request_id: result.request_id, - accepted: result.accepted, - generation: result.generation, - state_id: result.state_id, - reason: result.reason, - }); - Ok(true) + return Ok((true, false)); + } + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[], + chains: &[], + contacts: &[], + authorities: &[], + delete_controls: core::slice::from_ref(control), + presentation_changed: true, + }), + rng, + )?; + self.accept_commit_receipt( + receipt, + [Event::GroupAdminRequestResolved { + group: result.group, + request_id: result.request_id, + accepted: result.accepted, + generation: result.generation, + state_id: result.state_id, + reason: result.reason, + }], + ); + Ok((true, true)) } } diff --git a/crates/kult-node/src/devices.rs b/crates/kult-node/src/devices.rs index 69085a4..fed513b 100644 --- a/crates/kult-node/src/devices.rs +++ b/crates/kult-node/src/devices.rs @@ -47,7 +47,7 @@ pub(crate) fn initialize_fresh_device( store: &Store, account: &Identity, rng: &mut impl CryptoRngCore, -) -> Result<()> { +) -> kult_store::Result<()> { let device = Identity::generate(rng); let certificate = DeviceCertificate::issue(account, &device, 0, rng); let manifest = diff --git a/crates/kult-node/src/groups.rs b/crates/kult-node/src/groups.rs index 5dbcbf0..651f883 100644 --- a/crates/kult-node/src/groups.rs +++ b/crates/kult-node/src/groups.rs @@ -20,14 +20,20 @@ use kult_protocol::{ encode_edit, encode_ephemeral, encode_mention, encode_text, epoch_day, pad, retention_bucket, unpad, DecodedContent, DecodedGroupAuthority, Edit, Envelope, EnvelopeKind, Ephemeral, GroupAnnounce, GroupAuthorityAnnounce, GroupControlPayload, GroupMemberInfo, MailboxKey, Poll, - CONTENT_FORMAT_V1, CONTENT_KIND_EDIT, CONTENT_KIND_EPHEMERAL, CONTENT_KIND_MENTION, - MAX_EDIT_TEXT_LEN, MAX_EPHEMERAL_LIFETIME_SECS, MIN_EPHEMERAL_LIFETIME_SECS, + ReceiptPayload, CONTENT_FORMAT_V1, CONTENT_KIND_EDIT, CONTENT_KIND_EPHEMERAL, + CONTENT_KIND_MENTION, MAX_EDIT_TEXT_LEN, MAX_EPHEMERAL_LIFETIME_SECS, + MAX_GROUP_AUTHORITY_MEMBERS, MAX_GROUP_MEMBER_IDENTITY_LEN, MIN_EPHEMERAL_LIFETIME_SECS, }; use kult_store::{ - ContactDeviceRecord, ContactRecord, DeliveryState, Direction, EphemeralConversation, - EphemeralMode, EphemeralRecord, EphemeralState, GroupDelivery, GroupMember, GroupMessageRecord, - GroupRecord, MessageDeviceDeliveryRecord, PairwiseSendPlan, PendingAnnounce, QueueClass, - QueueItem, SessionTransition, + ContactDeviceRecord, ContactRecord, ContactTransition, DeferredControlRecord, DeliveryState, + DeliveryTransition, Direction, EphemeralConversation, EphemeralMode, EphemeralRecord, + EphemeralState, GroupAuthorityRecord, GroupAuthorityStateTransition, GroupAuthorityTransition, + GroupChainStateTransition, GroupChainTransition, GroupDelivery, GroupMember, + GroupMessageRecord, GroupMessageTransition, GroupReceivePlan, GroupRecord, GroupSendPlan, + GroupStatePlan, GroupStateTransition, GroupTransition, MediaDirection, MediaRecord, MediaScope, + MediaTransferState, MediaTransferTransition, MessageDeviceDeliveryRecord, PairwiseSendPlan, + PendingAnnounce, PendingDelete, QueueClass, QueueItem, ScheduledMessageRecord, + SessionTransition, }; use crate::api::{ @@ -56,6 +62,49 @@ struct DeviceGroupReceiverChain { chain: GroupReceiverChain, } +#[derive(Default)] +struct PreparedGroupCopy { + queue: Vec, + deliveries: Vec, + delivery_updates: Vec<(MessageDeviceDeliveryRecord, MessageDeviceDeliveryRecord)>, + first_wire: Option<[u8; 16]>, + all_served: bool, +} + +struct PreparedGroupControl { + preferred_wire: [u8; 16], + routes: Vec, + queue: Vec, +} + +struct PreparedGroupControlRoute { + device: [u8; 32], + before: Option, + after: Session, + reset: bool, +} + +type PreparedGroupReceiverChainUpdate = (Option>>, Option>); + +struct PreparedGroupInbound { + message: Option, + ephemeral: Option, + media_transfers: Vec, + media_objects: Vec, + events: Vec, + attachment_updates: Vec<[u8; 16]>, +} + +pub(crate) struct PreparedAuthorityGroupState { + pub(crate) secret: [u8; 32], + pub(crate) name: String, + pub(crate) creator: [u8; 32], + pub(crate) generation: u64, + pub(crate) members: Vec, + pub(crate) authority_before: Option, + pub(crate) authority_after: GroupAuthorityRecord, +} + fn decode_device_group_chains( bytes: &[u8], legacy_device: [u8; 32], @@ -126,6 +175,9 @@ impl Node { peer: *peer, identity: contact.identity, }); + if roster.len() > MAX_GROUP_AUTHORITY_MEMBERS { + return Err(NodeError::InvalidGroupAuthority); + } } let mut id = [0u8; 32]; @@ -134,22 +186,33 @@ impl Node { rng.fill_bytes(&mut secret); let chain = GroupSenderChain::generate(rng); let pending = pending_for(&chain, roster.iter().map(|m| m.peer), &me); - self.store.put_group( - &GroupRecord { - id, - name: name.to_owned(), - creator: me, - members: roster, - secret, - prev_secret: None, - generation: 1, - sender_chain: encode_chain(&chain)?, - sent_since_rotation: 0, - pending, - }, + let group = GroupRecord { + id, + name: name.to_owned(), + creator: me, + members: roster, + secret, + prev_secret: None, + generation: 1, + sender_chain: encode_chain(&chain)?, + sent_since_rotation: 0, + pending, + }; + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[GroupStateTransition { + before: None, + after: Some(&group), + }], + chains: &[], + contacts: &[], + authorities: &[], + delete_controls: &[], + presentation_changed: true, + }), rng, )?; - self.events.push_back(Event::GroupUpdated { group: id }); + self.accept_commit_receipt(receipt, [Event::GroupUpdated { group: id }]); Ok(id) } @@ -218,6 +281,38 @@ impl Node { timestamp: u64, now: u64, rng: &mut impl CryptoRngCore, + ) -> Result<[u8; 16]> { + self.group_send_with_id_source(group, body, id, timestamp, now, None, rng) + } + + pub(crate) fn activate_scheduled_group_message( + &mut self, + group: &[u8; 32], + scheduled: &ScheduledMessageRecord, + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result<[u8; 16]> { + self.group_send_with_id_source( + group, + &scheduled.body, + scheduled.id, + scheduled.not_before, + now, + Some(scheduled), + rng, + ) + } + + #[allow(clippy::too_many_arguments)] + fn group_send_with_id_source( + &mut self, + group: &[u8; 32], + body: &[u8], + id: [u8; 16], + timestamp: u64, + now: u64, + scheduled: Option<&ScheduledMessageRecord>, + rng: &mut impl CryptoRngCore, ) -> Result<[u8; 16]> { let rec = self .store @@ -239,7 +334,16 @@ impl Node { } else { body.to_vec() }; - self.group_send_content_with_id(group, wire_content, id, timestamp, now, rng) + self.group_send_content_with_id_retention( + group, + wire_content, + id, + timestamp, + now, + None, + scheduled, + rng, + ) } /// Current exact Mention capability intersection and review binding. @@ -453,18 +557,6 @@ impl Node { rng.fill_bytes(&mut id); let payload = encode_disappearing_text_payload(expires_at, text)?; let wire_content = encode_ephemeral(id, &payload)?; - self.store.put_ephemeral_record( - &EphemeralRecord { - conversation: EphemeralConversation::Group(*group), - author: me, - content_id: id, - expires_at, - mode: EphemeralMode::DisappearingText, - state: EphemeralState::Active, - transfer_ids: Vec::new(), - }, - rng, - )?; self.group_send_content_with_id_retention( group, wire_content, @@ -472,6 +564,7 @@ impl Node { now, now, Some(retention_until), + None, rng, ) } @@ -492,6 +585,7 @@ impl Node { timestamp, now, None, + None, rng, ) } @@ -505,12 +599,64 @@ impl Node { timestamp: u64, now: u64, retention_until: Option, + scheduled: Option<&ScheduledMessageRecord>, + rng: &mut impl CryptoRngCore, + ) -> Result<[u8; 16]> { + self.group_send_content_with_id_effects( + group, + wire_content, + id, + timestamp, + now, + retention_until, + scheduled, + None, + rng, + ) + } + + #[allow(clippy::too_many_arguments)] // authority message and resulting signed state + pub(crate) fn group_send_authority_content_with_id( + &mut self, + group: &[u8; 32], + wire_content: Vec, + id: [u8; 16], + timestamp: u64, + now: u64, + authority: &PreparedAuthorityGroupState, + rng: &mut impl CryptoRngCore, + ) -> Result<[u8; 16]> { + self.group_send_content_with_id_effects( + group, + wire_content, + id, + timestamp, + now, + None, + None, + Some(authority), + rng, + ) + } + + #[allow(clippy::too_many_arguments)] // canonical group send and exact durable effects + fn group_send_content_with_id_effects( + &mut self, + group: &[u8; 32], + wire_content: Vec, + id: [u8; 16], + timestamp: u64, + now: u64, + retention_until: Option, + scheduled: Option<&ScheduledMessageRecord>, + authority: Option<&PreparedAuthorityGroupState>, rng: &mut impl CryptoRngCore, ) -> Result<[u8; 16]> { - let mut rec = self + let before_group = self .store .get_group(group)? .ok_or(NodeError::UnknownGroup)?; + let mut after_group = before_group.clone(); let me = self.identity.public().ed; let mut record = GroupMessageRecord { id, @@ -519,7 +665,7 @@ impl Node { direction: Direction::Outbound, timestamp, body: wire_content.clone(), - deliveries: rec + deliveries: after_group .members .iter() .filter(|m| m.peer != me) @@ -531,44 +677,149 @@ impl Node { .collect(), wire_body: None, }; - self.store.put_group_message(&record, rng)?; - for d in &record.deliveries { - self.events.push_back(Event::GroupDeliveryUpdated { - id, - peer: d.peer, - state: DeliveryState::Queued, - }); - } // A spent chain rotates before it encrypts anything else (PCS). - if rec.sent_since_rotation >= GROUP_ROTATE_MSGS { - self.rotate_group(&mut rec, rng)?; + if after_group.sent_since_rotation >= GROUP_ROTATE_MSGS { + self.rotate_group(&mut after_group, rng)?; } - let mut chain = decode_chain(&rec.sender_chain)?; - let hk = GroupHeaderKey::derive(&rec.secret); - let wire = chain.seal(&hk, group, &pad(&wire_content)?, rng).encode(); - rec.sender_chain = encode_chain(&chain)?; - rec.sent_since_rotation += 1; + let mut chain = decode_chain(&after_group.sender_chain)?; + let hk = GroupHeaderKey::derive(&after_group.secret); + let wire = self + .candidate_group_seal(&mut chain, &hk, group, &pad(&wire_content)?, rng)? + .encode(); + after_group.sender_chain = encode_chain(&chain)?; + after_group.sent_since_rotation = after_group + .sent_since_rotation + .checked_add(1) + .ok_or(NodeError::CorruptState)?; + + let mut removed_peers = HashSet::new(); + if let Some(authority) = authority { + if authority.authority_after.group != *group + || authority.authority_after.state_id != id + || authority.members.is_empty() + { + return Err(NodeError::InvalidGroupAuthority); + } + let retained = authority + .members + .iter() + .map(|member| member.peer) + .collect::>(); + removed_peers.extend( + after_group + .members + .iter() + .filter(|member| !retained.contains(&member.peer)) + .map(|member| member.peer), + ); + after_group.prev_secret = Some(after_group.secret); + after_group.secret = authority.secret; + after_group.name.clone_from(&authority.name); + after_group.creator = authority.creator; + after_group.generation = authority.generation; + after_group.members.clone_from(&authority.members); + self.rotate_group(&mut after_group, rng)?; + } - let mut unserved = false; + let mut queue = Vec::new(); + let mut delivery_rows = Vec::new(); + let mut delivery_pairs = Vec::new(); + let mut unserved = record.deliveries.is_empty(); for d in record.deliveries.iter_mut() { - match self.queue_group_copy( + let prepared = self.prepare_group_copy( &d.peer, &wire, id, QueueClass::Interactive, retention_until, now, - rng, - )? { - Some(cid) => d.wire_id = Some(cid), - None => unserved = true, + )?; + if prepared.all_served { + d.wire_id = prepared.first_wire; + } else { + unserved = true; } + queue.extend(prepared.queue); + delivery_rows.extend(prepared.deliveries); + delivery_pairs.extend(prepared.delivery_updates); } record.wire_body = unserved.then_some(wire); - self.store.update_group_message(&record, rng)?; - self.store.put_group(&rec, rng)?; + let delivery_updates = delivery_pairs + .iter() + .map(|(before, after)| DeliveryTransition { before, after }) + .collect::>(); + let ephemeral = match decode_content(&wire_content) { + DecodedContent::Ephemeral { + id: content_id, + ephemeral: Ephemeral::DisappearingText { expires_at, .. }, + } => Some(EphemeralRecord { + conversation: EphemeralConversation::Group(*group), + author: me, + content_id, + expires_at, + mode: EphemeralMode::DisappearingText, + state: EphemeralState::Active, + transfer_ids: Vec::new(), + }), + _ => None, + }; + let deleted_chain_rows = self + .store + .group_chains(group)? + .into_iter() + .filter(|(peer, _)| removed_peers.contains(peer)) + .collect::>(); + let delete_chains = deleted_chain_rows + .iter() + .map(|(peer, chain)| GroupChainStateTransition { + group: *group, + peer: *peer, + before: Some(chain.as_slice()), + after: None, + }) + .collect::>(); + let authority_transition = authority.map(|authority| GroupAuthorityTransition { + before: authority.authority_before.as_ref(), + after: &authority.authority_after, + }); + let receipt = self.store.commit_plan( + CommitPlan::GroupSend(GroupSendPlan { + group: Some(GroupTransition { + before: &before_group, + after: &after_group, + }), + message: Some(&record), + message_update: None, + deliveries: &delivery_rows, + delivery_updates: &delivery_updates, + queue: &queue, + scheduled, + ephemeral: ephemeral.as_ref(), + media_transfers: &[], + delete_chains: &delete_chains, + authority: authority_transition, + presentation_changed: true, + }), + rng, + )?; + let mut events = record + .deliveries + .iter() + .map(|delivery| Event::GroupDeliveryUpdated { + id, + peer: delivery.peer, + state: DeliveryState::Queued, + }) + .collect::>(); + if scheduled.is_some() { + events.push(Event::ScheduledMessageActivated { id }); + } + if authority.is_some() { + events.push(Event::GroupUpdated { group: *group }); + } + self.accept_commit_receipt(receipt, events); Ok(id) } @@ -582,10 +833,11 @@ impl Node { now: u64, rng: &mut impl CryptoRngCore, ) -> Result<()> { - let mut rec = self + let before = self .store .get_group(group)? .ok_or(NodeError::UnknownGroup)?; + let mut rec = before.clone(); let me = self.identity.public().ed; if self.store.get_group_authority(group)?.is_some() { if rec.members.iter().any(|member| &member.peer == peer) { @@ -612,6 +864,9 @@ impl Node { if rec.members.iter().any(|m| &m.peer == peer) { return Ok(()); // already in — idempotent } + if rec.members.len() >= MAX_GROUP_AUTHORITY_MEMBERS { + return Err(NodeError::InvalidGroupAuthority); + } let contact = self .store .get_contact(peer)? @@ -640,8 +895,21 @@ impl Node { last_sent: 0, }); } - self.store.put_group(&rec, rng)?; - self.events.push_back(Event::GroupUpdated { group: *group }); + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[GroupStateTransition { + before: Some(&before), + after: Some(&rec), + }], + chains: &[], + contacts: &[], + authorities: &[], + delete_controls: &[], + presentation_changed: true, + }), + rng, + )?; + self.accept_commit_receipt(receipt, [Event::GroupUpdated { group: *group }]); Ok(()) } @@ -663,10 +931,11 @@ impl Node { self.group_authority_remove_member(group, *peer, now, rng)?; return Ok(()); } - let mut rec = self + let before = self .store .get_group(group)? .ok_or(NodeError::UnknownGroup)?; + let mut rec = before.clone(); if rec.creator != me { return Err(NodeError::NotGroupCreator); } @@ -674,12 +943,36 @@ impl Node { return Err(NodeError::UnknownPeer); } rec.members.retain(|m| &m.peer != peer); - self.store.delete_group_chain(group, peer)?; + let chain_before = self.store.get_group_chain(group, peer)?; rec.generation += 1; rec.prev_secret = Some(rec.secret); rng.fill_bytes(&mut rec.secret); self.rotate_group(&mut rec, rng)?; // also drops the removed peer's pending entry - self.store.put_group(&rec, rng)?; + let chain_transitions = chain_before + .as_ref() + .map(|chain| GroupChainStateTransition { + group: *group, + peer: *peer, + before: Some(chain.as_slice()), + after: None, + }) + .into_iter() + .collect::>(); + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[GroupStateTransition { + before: Some(&before), + after: Some(&rec), + }], + chains: &chain_transitions, + contacts: &[], + authorities: &[], + delete_controls: &[], + presentation_changed: true, + }), + rng, + )?; + self.accept_commit_receipt(receipt, [Event::GroupUpdated { group: *group }]); // Best effort: keys are already rotated whether or not this lands. self.queue_group_control( peer, @@ -687,7 +980,6 @@ impl Node { now, rng, )?; - self.events.push_back(Event::GroupUpdated { group: *group }); Ok(()) } @@ -705,11 +997,45 @@ impl Node { .get_group(group)? .ok_or(NodeError::UnknownGroup)?; let me = self.identity.public().ed; - if self.store.get_group_authority(group)?.is_some() + let authority = self.store.get_group_authority(group)?; + if authority.is_some() && self.group_authority(group)?.my_role == Some(kult_protocol::GroupRole::Owner) { return Err(NodeError::LastGroupOwner); } + let chain_rows = self.store.group_chains(group)?; + let chain_transitions = chain_rows + .iter() + .map(|(peer, chain)| GroupChainStateTransition { + group: *group, + peer: *peer, + before: Some(chain.as_slice()), + after: None, + }) + .collect::>(); + let authority_transitions = authority + .as_ref() + .map(|authority| GroupAuthorityStateTransition { + before: Some(authority), + after: None, + }) + .into_iter() + .collect::>(); + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[GroupStateTransition { + before: Some(&rec), + after: None, + }], + chains: &chain_transitions, + contacts: &[], + authorities: &authority_transitions, + delete_controls: &[], + presentation_changed: true, + }), + rng, + )?; + self.accept_commit_receipt(receipt, [Event::GroupUpdated { group: *group }]); for member in &rec.members { if member.peer == me { continue; @@ -721,8 +1047,6 @@ impl Node { rng, )?; } - self.store.delete_group(group)?; - self.events.push_back(Event::GroupUpdated { group: *group }); Ok(()) } @@ -746,9 +1070,8 @@ impl Node { .collect(); for mut rec in self.store.groups()? { - let mut dirty = false; - let mut pending = std::mem::take(&mut rec.pending); - for entry in pending.iter_mut() { + for pending_index in 0..rec.pending.len() { + let entry = rec.pending[pending_index].clone(); // Due when never attempted, or when the retry window passed // and the last envelope is out of the queue (a queued one is // still the transport scheduler's problem, not ours). @@ -797,49 +1120,82 @@ impl Node { iteration: entry.iteration, }), }; - entry.wire_id = self.queue_group_control(&entry.peer, &announce, now, rng)?; - entry.last_sent = now; // paces the next attempt either way - dirty = true; - } - rec.pending = pending; - if dirty { - self.store.put_group(&rec, rng)?; + let before = rec.clone(); + self.queue_group_announce( + &entry.peer, + &announce, + &before, + &mut rec, + pending_index, + now, + rng, + )?; } } // Late fan-out from retained ciphertexts. - for mut record in self.store.all_group_messages()? { - let Some(wire) = record.wire_body.clone() else { + for before_record in self.store.all_group_messages()? { + let Some(wire) = before_record.wire_body.clone() else { continue; }; + let mut record = before_record.clone(); let mut changed = false; let mut unserved = false; + let mut queue = Vec::new(); + let mut delivery_rows = Vec::new(); + let mut delivery_pairs = Vec::new(); for d in record.deliveries.iter_mut() { if d.wire_id.is_some() { continue; } - match self.queue_group_copy( + let prepared = self.prepare_group_copy( &d.peer, &wire, record.id, QueueClass::Interactive, ephemeral_retention(&record.body), now, - rng, - )? { - Some(cid) => { - d.wire_id = Some(cid); - changed = true; - } - None => unserved = true, + )?; + if prepared.all_served { + d.wire_id = prepared.first_wire; + changed = true; + } else { + unserved = true; } + queue.extend(prepared.queue); + delivery_rows.extend(prepared.deliveries); + delivery_pairs.extend(prepared.delivery_updates); } if !unserved { record.wire_body = None; changed = true; } - if changed { - self.store.update_group_message(&record, rng)?; + if changed || !queue.is_empty() { + let delivery_updates = delivery_pairs + .iter() + .map(|(before, after)| DeliveryTransition { before, after }) + .collect::>(); + let receipt = self.store.commit_plan( + CommitPlan::GroupSend(GroupSendPlan { + group: None, + message: None, + message_update: Some(GroupMessageTransition { + before: &before_record, + after: &record, + }), + deliveries: &delivery_rows, + delivery_updates: &delivery_updates, + queue: &queue, + scheduled: None, + ephemeral: None, + media_transfers: &[], + delete_chains: &[], + authority: None, + presentation_changed: false, + }), + rng, + )?; + self.accept_commit_receipt(receipt, []); } } Ok(()) @@ -912,13 +1268,61 @@ impl Node { } if changed { let encoded = encode_device_group_chains(&mut chains)?; - self.store - .put_group_chain(&group.id, account, &encoded, rng)?; + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[], + chains: &[GroupChainStateTransition { + group: group.id, + peer: *account, + before: Some(blob.as_slice()), + after: Some(&encoded), + }], + contacts: &[], + authorities: &[], + delete_controls: &[], + presentation_changed: false, + }), + rng, + )?; + self.accept_commit_receipt(receipt, []); } } Ok(()) } + pub(crate) fn prepare_group_receiver_chain( + &self, + group: &[u8; 32], + peer: [u8; 32], + peer_device: [u8; 32], + key_id: [u8; 16], + chain_key: &[u8; 32], + iteration: u32, + ) -> Result { + let before = self.store.get_group_chain(group, &peer)?; + let mut chains = match before.as_ref() { + Some(blob) => decode_device_group_chains(blob, peer)?, + None => Vec::new(), + }; + let replace = chains + .iter() + .find(|entry| entry.device == peer_device) + .is_none_or(|entry| entry.chain.key_id() != key_id); + if !replace { + return Ok((before, None)); + } + let chain = GroupReceiverChain::new(key_id, chain_key, iteration); + if let Some(entry) = chains.iter_mut().find(|entry| entry.device == peer_device) { + entry.chain = chain; + } else { + chains.push(DeviceGroupReceiverChain { + device: peer_device, + chain, + }); + } + Ok((before, Some(encode_device_group_chains(&mut chains)?))) + } + // ---- receive path -------------------------------------------------------- /// Consume a `GroupMessage` envelope. The delivery token names the @@ -929,418 +1333,509 @@ impl Node { pub(crate) fn consume_group_message( &mut self, env: &Envelope, + pending_sequence: Option, now: u64, rng: &mut impl CryptoRngCore, - acks: &mut Vec<([u8; 32], [u8; 16])>, ) -> Result { let Some(peer_device) = self.match_session(&env.token, now) else { return Ok(Consumed::Later); }; let peer = self.account_for_device(&peer_device)?; - let me = self.identity.public().ed; - let done = |node: &mut Self| -> Result { - node.store.mark_seen(&env.content_id())?; - Ok(Consumed::Done) - }; - let Ok(msg) = GroupMessage::decode(&env.body) else { - return done(self); + let Ok(message) = GroupMessage::decode(&env.body) else { + return self.commit_terminal_input(env.content_id(), pending_sequence, rng); }; - for rec in self.store.groups()? { - if !rec.members.iter().any(|m| m.peer == peer) { + for group in self.store.groups()? { + if !group.members.iter().any(|member| member.peer == peer) { continue; } - // Current header key first; the previous one covers in-flight - // traffic crossing a re-key (kept one generation deep). let mut opened = None; - for secret in core::iter::once(rec.secret).chain(rec.prev_secret) { - let hk = GroupHeaderKey::derive(&secret); - if let Ok(header) = msg.open_header(&hk) { - opened = Some(header); + for secret in core::iter::once(group.secret).chain(group.prev_secret) { + let header = GroupHeaderKey::derive(&secret); + if let Ok(value) = message.open_header(&header) { + opened = Some(value); break; } } let Some((key_id, iteration)) = opened else { continue; }; - let Some(blob) = self.store.get_group_chain(&rec.id, &peer)? else { - return Ok(Consumed::Later); // sender's announce still in flight + let Some(before_blob) = self.store.get_group_chain(&group.id, &peer)? else { + return Ok(Consumed::Later); }; - let mut chains = decode_device_group_chains(&blob, peer)?; + let mut chains = decode_device_group_chains(&before_blob, peer)?; let Some(chain) = chains .iter_mut() .find(|entry| entry.device == peer_device && entry.chain.key_id() == key_id) .map(|entry| &mut entry.chain) else { - return Ok(Consumed::Later); // rotation announce still in flight + return Ok(Consumed::Later); }; - let Ok(padded) = chain.open(&rec.id, &msg, iteration, now) else { - // Tampered or replayed — permanent, honest failure. - return done(self); - }; - let encoded = encode_device_group_chains(&mut chains)?; - self.store.put_group_chain(&rec.id, &peer, &encoded, rng)?; - let Ok(body) = unpad(&padded) else { - return done(self); + let padded = + match self.candidate_group_open(chain, &group.id, &message, iteration, now)? { + Ok(plaintext) => plaintext, + Err(_) => { + return self.commit_terminal_input(env.content_id(), pending_sequence, rng) + } + }; + let after_blob = encode_device_group_chains(&mut chains)?; + let prepared = match unpad(&padded) { + Ok(body) => { + self.prepare_group_inbound(&group, peer, body, env.retention_until, now, rng)? + } + Err(_) => PreparedGroupInbound { + message: None, + ephemeral: None, + media_transfers: Vec::new(), + media_objects: Vec::new(), + events: Vec::new(), + attachment_updates: Vec::new(), + }, }; - let decoded = decode_content(&body); - let authenticated_retention = match decoded { - DecodedContent::Ephemeral { ephemeral, .. } => Some(match ephemeral { - Ephemeral::DisappearingText { - retention_until, .. - } - | Ephemeral::ViewOnceAttachment { - retention_until, .. - } => retention_until, + let receipt_before = self + .store + .get_session(&peer_device)? + .ok_or(NodeError::CorruptState)?; + let mut receipt_after = receipt_before.clone(); + let receipt_payload = ReceiptPayload { + acks: vec![env.content_id()], + nacks: Vec::new(), + } + .encode(); + let receipt_queue = self.prepare_control_queue( + &mut receipt_after, + peer_device, + &receipt_payload, + now, + rng, + )?; + let source_pending = pending_sequence.map(|sequence| PendingDelete { + sequence, + content_id: env.content_id(), + }); + let presentation_changed = !prepared.events.is_empty() + || !prepared.attachment_updates.is_empty() + || prepared.ephemeral.is_some(); + let receipt = self.store.commit_plan( + CommitPlan::GroupReceive(GroupReceivePlan { + chain: GroupChainTransition { + group: group.id, + peer, + before: &before_blob, + after: &after_blob, + }, + receipt_session: SessionTransition { + peer_device, + before: Some(&receipt_before), + after: &receipt_after, + }, + message: prepared.message.as_ref(), + ephemeral: prepared.ephemeral.as_ref(), + media_transfers: &prepared.media_transfers, + media_objects: &prepared.media_objects, + queue: &[receipt_queue], + content_id: env.content_id(), + received_at: now, + source_pending, + presentation_changed, }), - _ => None, - }; - if env.retention_until != authenticated_retention { - return done(self); + rng, + )?; + self.before_memory_replacement()?; + self.sessions.insert(peer_device, receipt_after); + self.after_memory_replacement()?; + self.accept_commit_receipt(receipt, prepared.events); + for transfer in prepared.attachment_updates { + self.emit_attachment_update(&transfer)?; } - let decoded_is_edit = matches!(decoded, DecodedContent::Edit { .. }); - if let DecodedContent::Text { id, .. } - | DecodedContent::Attachment { id, .. } - | DecodedContent::Mention { id, .. } - | DecodedContent::Edit { id, .. } - | DecodedContent::Ephemeral { id, .. } - | DecodedContent::Poll { id, .. } - | DecodedContent::GroupAuthority { id, .. } = decoded - { - let conversation = EphemeralConversation::Group(rec.id); - if self - .store - .get_ephemeral_record(&conversation, &peer, &id)? - .is_some() - { - acks.push((peer_device, env.content_id())); - return done(self); + return Ok(Consumed::DoneAtomic); + } + Ok(Consumed::Later) + } + + fn prepare_group_inbound( + &self, + group: &GroupRecord, + peer: [u8; 32], + body: Vec, + envelope_retention: Option, + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result { + let empty = || PreparedGroupInbound { + message: None, + ephemeral: None, + media_transfers: Vec::new(), + media_objects: Vec::new(), + events: Vec::new(), + attachment_updates: Vec::new(), + }; + let decoded = decode_content(&body); + let authenticated_retention = match decoded { + DecodedContent::Ephemeral { ephemeral, .. } => Some(match ephemeral { + Ephemeral::DisappearingText { + retention_until, .. } - let duplicate = self.store.group_messages(&rec.id)?.iter().any(|record| { + | Ephemeral::ViewOnceAttachment { + retention_until, .. + } => retention_until, + }), + _ => None, + }; + if authenticated_retention != envelope_retention { + return Ok(empty()); + } + if let DecodedContent::Text { id, .. } + | DecodedContent::Attachment { id, .. } + | DecodedContent::Mention { id, .. } + | DecodedContent::Edit { id, .. } + | DecodedContent::Ephemeral { id, .. } + | DecodedContent::Poll { id, .. } + | DecodedContent::GroupAuthority { id, .. } = decoded + { + let conversation = EphemeralConversation::Group(group.id); + if self + .store + .get_ephemeral_record(&conversation, &peer, &id)? + .is_some() + || self.store.group_messages(&group.id)?.iter().any(|record| { record.direction == Direction::Inbound && record.sender == peer && matches!( decode_content(&record.body), - DecodedContent::Text { id: stored_id, .. } - | DecodedContent::Attachment { id: stored_id, .. } - | DecodedContent::Mention { id: stored_id, .. } - | DecodedContent::Edit { id: stored_id, .. } - | DecodedContent::Ephemeral { id: stored_id, .. } - | DecodedContent::Poll { id: stored_id, .. } - | DecodedContent::GroupAuthority { id: stored_id, .. } - if stored_id == id + DecodedContent::Text { id: stored, .. } + | DecodedContent::Attachment { id: stored, .. } + | DecodedContent::Mention { id: stored, .. } + | DecodedContent::Edit { id: stored, .. } + | DecodedContent::Ephemeral { id: stored, .. } + | DecodedContent::Poll { id: stored, .. } + | DecodedContent::GroupAuthority { id: stored, .. } + if stored == id ) - }); - if duplicate { - acks.push((peer_device, env.content_id())); - return done(self); - } + }) + { + return Ok(empty()); } - let mut mentions_local_peer = false; - let (id, event_body, content) = match decoded { - DecodedContent::LegacyText(text) => { - let mut id = [0u8; 16]; - rng.fill_bytes(&mut id); - (id, text.as_bytes().to_vec(), ContentStatus::LegacyText) - } - DecodedContent::Text { id, text } => { - (id, text.as_bytes().to_vec(), ContentStatus::Text { id }) - } - DecodedContent::Attachment { id, manifest } => { - let entitled_peers = rec.members.iter().map(|member| member.peer).collect(); - let transfer = self.record_group_attachment_offer( - crate::attachment::GroupAttachmentOffer { - group: rec.id, - author: peer, - entitled_peers, - }, - id, - &manifest, - now, - rng, - )?; - self.emit_attachment_update(&transfer)?; - (id, Vec::new(), ContentStatus::Attachment { id, transfer }) - } - DecodedContent::Mention { id, mention } => { - let spans = mention.spans().map(MentionSpan::from).collect::>(); - mentions_local_peer = spans.iter().any(|span| span.target == me); - ( - id, - mention.text.as_bytes().to_vec(), - ContentStatus::Mention { id, spans }, - ) - } - DecodedContent::Edit { id, edit } if edit.target_author == peer => ( - id, - Vec::new(), - ContentStatus::Edit { - id, - target_author: edit.target_author, - target_content_id: edit.target_content_id, - revision: edit.revision, - }, - ), - DecodedContent::Edit { id, .. } => (id, Vec::new(), ContentStatus::Malformed), - DecodedContent::Ephemeral { - id, - ephemeral: - Ephemeral::DisappearingText { - expires_at, text, .. - }, - } => { - let state = if now >= expires_at { - EphemeralState::Expired - } else { - EphemeralState::Active - }; - self.store.put_ephemeral_record( - &EphemeralRecord { - conversation: EphemeralConversation::Group(rec.id), - author: peer, - content_id: id, - expires_at, - mode: EphemeralMode::DisappearingText, - state, - transfer_ids: Vec::new(), - }, - rng, - )?; - if state == EphemeralState::Expired { - self.events.push_back(Event::EphemeralRemoved { - conversation: EphemeralConversation::Group(rec.id), - author: peer, - content_id: id, - reason: state, - }); - acks.push((peer_device, env.content_id())); - return done(self); - } - ( - id, - text.as_bytes().to_vec(), - ContentStatus::DisappearingText { id, expires_at }, - ) - } - DecodedContent::Poll { + } + + let me = self.identity.public().ed; + let decoded_is_edit = matches!(decoded, DecodedContent::Edit { .. }); + let mut ephemeral = None; + let mut media_transfers = Vec::new(); + let mut media_objects = Vec::new(); + let mut attachment_updates = Vec::new(); + let mut mentions_local_peer = false; + let (id, event_body, content) = match decoded { + DecodedContent::LegacyText(text) => { + let mut id = [0u8; 16]; + rng.fill_bytes(&mut id); + (id, text.as_bytes().to_vec(), ContentStatus::LegacyText) + } + DecodedContent::Text { id, text } => { + (id, text.as_bytes().to_vec(), ContentStatus::Text { id }) + } + DecodedContent::Attachment { id, manifest } => { + let (transfer, objects) = self.prepare_group_attachment_offer( + crate::attachment::GroupAttachmentOffer { + group: group.id, + author: peer, + entitled_peers: group.members.iter().map(|member| member.peer).collect(), + }, id, - poll: Poll::Create(create), - } if create.voters().any(|voter| voter == peer) => ( + &manifest, + now, + rng, + )?; + let transfer_id = transfer.local_id; + media_transfers.push(transfer); + media_objects.extend(objects); + attachment_updates.push(transfer_id); + ( id, Vec::new(), - ContentStatus::Poll { + ContentStatus::Attachment { id, - poll_author: peer, - poll_id: id, + transfer: transfer_id, }, - ), - DecodedContent::Poll { + ) + } + DecodedContent::Mention { id, mention } => { + let spans = mention.spans().map(MentionSpan::from).collect::>(); + mentions_local_peer = spans.iter().any(|span| span.target == me); + ( id, - poll: Poll::Vote(vote), - } => ( + mention.text.as_bytes().to_vec(), + ContentStatus::Mention { id, spans }, + ) + } + DecodedContent::Edit { id, edit } if edit.target_author == peer => ( + id, + Vec::new(), + ContentStatus::Edit { id, - Vec::new(), - ContentStatus::Poll { - id, - poll_author: vote.poll_author, - poll_id: vote.poll_id, + target_author: edit.target_author, + target_content_id: edit.target_content_id, + revision: edit.revision, + }, + ), + DecodedContent::Edit { id, .. } => (id, Vec::new(), ContentStatus::Malformed), + DecodedContent::Ephemeral { + id, + ephemeral: + Ephemeral::DisappearingText { + expires_at, text, .. }, - ), - DecodedContent::Poll { - id, - poll: Poll::Close(close), - } if close.poll_author == peer => ( + } => { + let state = if now >= expires_at { + EphemeralState::Expired + } else { + EphemeralState::Active + }; + ephemeral = Some(EphemeralRecord { + conversation: EphemeralConversation::Group(group.id), + author: peer, + content_id: id, + expires_at, + mode: EphemeralMode::DisappearingText, + state, + transfer_ids: Vec::new(), + }); + if state == EphemeralState::Expired { + return Ok(PreparedGroupInbound { + message: None, + ephemeral, + media_transfers, + media_objects, + events: vec![Event::EphemeralRemoved { + conversation: EphemeralConversation::Group(group.id), + author: peer, + content_id: id, + reason: state, + }], + attachment_updates, + }); + } + ( id, - Vec::new(), - ContentStatus::Poll { - id, - poll_author: close.poll_author, - poll_id: close.poll_id, + text.as_bytes().to_vec(), + ContentStatus::DisappearingText { id, expires_at }, + ) + } + DecodedContent::Ephemeral { + id, + ephemeral: + Ephemeral::ViewOnceAttachment { + expires_at, + manifest, + .. + }, + } => { + if now >= expires_at { + ephemeral = Some(EphemeralRecord { + conversation: EphemeralConversation::Group(group.id), + author: peer, + content_id: id, + expires_at, + mode: EphemeralMode::ViewOnceAttachment, + state: EphemeralState::Expired, + transfer_ids: Vec::new(), + }); + return Ok(PreparedGroupInbound { + message: None, + ephemeral, + media_transfers, + media_objects, + events: vec![Event::EphemeralRemoved { + conversation: EphemeralConversation::Group(group.id), + author: peer, + content_id: id, + reason: EphemeralState::Expired, + }], + attachment_updates, + }); + } + let (transfer, objects) = self.prepare_group_attachment_offer( + crate::attachment::GroupAttachmentOffer { + group: group.id, + author: peer, + entitled_peers: group.members.iter().map(|member| member.peer).collect(), }, - ), - DecodedContent::Poll { id, - poll: Poll::ModeratedClose(close), - } => ( + &manifest, + now, + rng, + )?; + let transfer_id = transfer.local_id; + media_transfers.push(transfer); + media_objects.extend(objects); + attachment_updates.push(transfer_id); + ephemeral = Some(EphemeralRecord { + conversation: EphemeralConversation::Group(group.id), + author: peer, + content_id: id, + expires_at, + mode: EphemeralMode::ViewOnceAttachment, + state: EphemeralState::Active, + transfer_ids: vec![transfer_id], + }); + ( id, Vec::new(), - ContentStatus::Poll { + ContentStatus::ViewOnceAttachment { id, - poll_author: close.poll_author, - poll_id: close.poll_id, + transfer: transfer_id, + expires_at, }, - ), - DecodedContent::Poll { id, .. } => (id, Vec::new(), ContentStatus::Malformed), - DecodedContent::GroupAuthority { id, payload } => { - match decode_group_authority(payload) { - DecodedGroupAuthority::State(state) - if state.group == rec.id - && state.signer == peer - && crate::authority::verify_authority_state(&state, None) - .is_ok() => - { - ( - id, - Vec::new(), - ContentStatus::GroupAuthority { - id, - generation: state.generation, - owner: state.owner, - }, - ) - } - _ => (id, Vec::new(), ContentStatus::Malformed), - } - } - // Call controls are pairwise-only; group delivery retains the - // authenticated bytes as malformed history without acting. - DecodedContent::CallControl { id, .. } => { - (id, Vec::new(), ContentStatus::Malformed) - } - DecodedContent::Ephemeral { + ) + } + DecodedContent::Poll { + id, + poll: Poll::Create(create), + } if create.voters().any(|voter| voter == peer) => ( + id, + Vec::new(), + ContentStatus::Poll { id, - ephemeral: - Ephemeral::ViewOnceAttachment { - expires_at, - manifest, - .. - }, - } => { - if now >= expires_at { - self.store.put_ephemeral_record( - &EphemeralRecord { - conversation: EphemeralConversation::Group(rec.id), - author: peer, - content_id: id, - expires_at, - mode: EphemeralMode::ViewOnceAttachment, - state: EphemeralState::Expired, - transfer_ids: Vec::new(), - }, - rng, - )?; - self.events.push_back(Event::EphemeralRemoved { - conversation: EphemeralConversation::Group(rec.id), - author: peer, - content_id: id, - reason: EphemeralState::Expired, - }); - acks.push((peer_device, env.content_id())); - return done(self); - } - let entitled_peers = rec.members.iter().map(|member| member.peer).collect(); - let transfer = self.record_group_attachment_offer( - crate::attachment::GroupAttachmentOffer { - group: rec.id, - author: peer, - entitled_peers, - }, - id, - &manifest, - now, - rng, - )?; - self.store.put_ephemeral_record( - &EphemeralRecord { - conversation: EphemeralConversation::Group(rec.id), - author: peer, - content_id: id, - expires_at, - mode: EphemeralMode::ViewOnceAttachment, - state: EphemeralState::Active, - transfer_ids: vec![transfer], - }, - rng, - )?; - self.emit_attachment_update(&transfer)?; + poll_author: peer, + poll_id: id, + }, + ), + DecodedContent::Poll { + id, + poll: Poll::Vote(vote), + } => ( + id, + Vec::new(), + ContentStatus::Poll { + id, + poll_author: vote.poll_author, + poll_id: vote.poll_id, + }, + ), + DecodedContent::Poll { + id, + poll: Poll::Close(close), + } if close.poll_author == peer => ( + id, + Vec::new(), + ContentStatus::Poll { + id, + poll_author: close.poll_author, + poll_id: close.poll_id, + }, + ), + DecodedContent::Poll { + id, + poll: Poll::ModeratedClose(close), + } => ( + id, + Vec::new(), + ContentStatus::Poll { + id, + poll_author: close.poll_author, + poll_id: close.poll_id, + }, + ), + DecodedContent::Poll { id, .. } => (id, Vec::new(), ContentStatus::Malformed), + DecodedContent::GroupAuthority { id, payload } => match decode_group_authority(payload) + { + DecodedGroupAuthority::State(state) + if state.group == group.id + && state.signer == peer + && crate::authority::verify_authority_state(&state, None).is_ok() => + { ( id, Vec::new(), - ContentStatus::ViewOnceAttachment { + ContentStatus::GroupAuthority { id, - transfer, - expires_at, + generation: state.generation, + owner: state.owner, }, ) } - DecodedContent::Unsupported { - format_version, - kind, - } => { - let mut id = [0u8; 16]; - rng.fill_bytes(&mut id); - ( - id, - Vec::new(), - ContentStatus::Unsupported { - format_version, - kind, - }, - ) - } - DecodedContent::Malformed => { - let mut id = [0u8; 16]; - rng.fill_bytes(&mut id); - (id, Vec::new(), ContentStatus::Malformed) - } - }; - self.store.put_group_message( - &GroupMessageRecord { - id, - group: rec.id, - sender: peer, - direction: Direction::Inbound, - timestamp: now, - body, - deliveries: Vec::new(), - wire_body: None, - }, - rng, - )?; - match content { - ContentStatus::Edit { - target_content_id, .. - } => self.events.push_back(Event::GroupMessageEdited { - group: rec.id, - sender: peer, - target_content_id, - }), - ContentStatus::Poll { - poll_author, - poll_id, - .. - } => self.events.push_back(Event::PollUpdated { - group: rec.id, - poll_author, - poll_id, - }), - ContentStatus::GroupAuthority { - generation, owner, .. - } => self.events.push_back(Event::GroupAuthorityUpdated { - group: rec.id, - generation, - owner, - }), - ContentStatus::Malformed if decoded_is_edit => {} - _ => self.events.push_back(Event::GroupMessageReceived { - group: rec.id, - sender: peer, + _ => (id, Vec::new(), ContentStatus::Malformed), + }, + DecodedContent::CallControl { id, .. } => (id, Vec::new(), ContentStatus::Malformed), + DecodedContent::Unsupported { + format_version, + kind, + } => { + let mut id = [0u8; 16]; + rng.fill_bytes(&mut id); + ( id, - timestamp: now, - body: event_body, - content, - }), + Vec::new(), + ContentStatus::Unsupported { + format_version, + kind, + }, + ) } - if mentions_local_peer { - self.events.push_back(Event::MentionReceived { id }); + DecodedContent::Malformed => { + let mut id = [0u8; 16]; + rng.fill_bytes(&mut id); + (id, Vec::new(), ContentStatus::Malformed) } - acks.push((peer_device, env.content_id())); - return done(self); + }; + + let message = GroupMessageRecord { + id, + group: group.id, + sender: peer, + direction: Direction::Inbound, + timestamp: now, + body, + deliveries: Vec::new(), + wire_body: None, + }; + let mut events = Vec::new(); + match content { + ContentStatus::Edit { + target_content_id, .. + } => events.push(Event::GroupMessageEdited { + group: group.id, + sender: peer, + target_content_id, + }), + ContentStatus::Poll { + poll_author, + poll_id, + .. + } => events.push(Event::PollUpdated { + group: group.id, + poll_author, + poll_id, + }), + ContentStatus::GroupAuthority { + generation, owner, .. + } => events.push(Event::GroupAuthorityUpdated { + group: group.id, + generation, + owner, + }), + ContentStatus::Malformed if decoded_is_edit => {} + _ => events.push(Event::GroupMessageReceived { + group: group.id, + sender: peer, + id, + timestamp: now, + body: event_body, + content, + }), } - // No group of this sender opened the header: the invite may still - // be in flight (or the message is junk — the pending TTL bounds it). - Ok(Consumed::Later) + if mentions_local_peer { + events.push(Event::MentionReceived { id }); + } + Ok(PreparedGroupInbound { + message: Some(message), + ephemeral, + media_transfers, + media_objects, + events, + attachment_updates, + }) } /// Apply a decrypted `GroupControl` payload from `peer`. Returns whether @@ -1349,37 +1844,44 @@ impl Node { /// co-member's announce racing the creator's invite). pub(crate) fn apply_group_control( &mut self, - peer: [u8; 32], - peer_device: [u8; 32], - body: &[u8], + control: &DeferredControlRecord, now: u64, rng: &mut impl CryptoRngCore, established: &mut bool, - ) -> Result { - let Ok(payload) = GroupControlPayload::decode(body) else { - return Ok(true); // malformed is terminal — ack so it is not resent + ) -> Result<(bool, bool)> { + let Ok(payload) = GroupControlPayload::decode(&control.body) else { + return Ok((true, false)); // malformed is terminal }; - let _ = now; + let peer = control.peer; match &payload { GroupControlPayload::Announce(a) => { - self.apply_group_announce(peer, peer_device, a, rng, established) + self.apply_group_announce(peer, control.peer_device, a, control, rng, established) } - GroupControlPayload::Leave { group } => self.apply_group_leave(peer, group, rng), - GroupControlPayload::Remove { group } => { - self.apply_group_remove_notice(peer, group, rng) + GroupControlPayload::Leave { group } => { + self.apply_group_leave(peer, group, control, rng) } - GroupControlPayload::AuthorityAnnounce(announce) => { - self.apply_authority_announce(peer, announce, rng, established) + GroupControlPayload::Remove { group } => { + self.apply_group_remove_notice(peer, group, control, rng) } + GroupControlPayload::AuthorityAnnounce(announce) => self.apply_authority_announce( + peer, + control.peer_device, + announce, + control, + rng, + established, + ), GroupControlPayload::AdminRequest(request) => { - self.apply_group_admin_request(peer, request, now, rng) + self.apply_group_admin_request(peer, request, control, now, rng) + } + GroupControlPayload::AdminResult(result) => { + self.apply_group_admin_result(peer, result, control, rng) } - GroupControlPayload::AdminResult(result) => self.apply_group_admin_result(peer, result), GroupControlPayload::AuthorityRemove { group, state_id, state_payload, - } => self.apply_authority_remove(peer, group, state_id, state_payload), + } => self.apply_authority_remove(peer, group, state_id, state_payload, control, rng), } } @@ -1388,21 +1890,26 @@ impl Node { peer: [u8; 32], peer_device: [u8; 32], a: &GroupAnnounce, + control: &DeferredControlRecord, rng: &mut impl CryptoRngCore, established: &mut bool, - ) -> Result { + ) -> Result<(bool, bool)> { let me = self.identity.public().ed; - let rec = match self.store.get_group(&a.group)? { + let before_group = self.store.get_group(&a.group)?; + let mut group_changed = false; + let mut contact_records = Vec::new(); + let rec = match before_group.as_ref() { None => { // An invite: only the claimed creator's announce creates // the group, and it must list both of us. if a.creator != peer || !a.members.iter().any(|m| m.peer == me) || !a.members.iter().any(|m| m.peer == peer) + || !valid_roster(&a.members) { - return Ok(false); + return Ok((false, false)); } - self.adopt_roster_stubs(&a.members, rng)?; + contact_records = self.prepare_roster_stubs(&a.members)?; let chain = GroupSenderChain::generate(rng); let members: Vec = a .members @@ -1425,26 +1932,24 @@ impl Node { sent_since_rotation: 0, pending, }; - self.events - .push_back(Event::GroupUpdated { group: a.group }); + group_changed = true; rec } - Some(mut rec) => { + Some(stored) => { + let mut rec = stored.clone(); if peer == rec.creator && a.generation > rec.generation { if !a.members.iter().any(|m| m.peer == me) { // The creator's newer roster omits us: removed. - self.store.delete_group(&rec.id)?; - self.events - .push_back(Event::GroupUpdated { group: a.group }); - return Ok(true); + self.commit_group_removal(&rec, control, rng)?; + return Ok((true, true)); + } + if !valid_roster(&a.members) { + return Ok((true, false)); } let old: HashSet<[u8; 32]> = rec.members.iter().map(|m| m.peer).collect(); let new: HashSet<[u8; 32]> = a.members.iter().map(|m| m.peer).collect(); - for gone in old.difference(&new) { - self.store.delete_group_chain(&rec.id, gone)?; - } rec.pending.retain(|p| new.contains(&p.peer)); - self.adopt_roster_stubs(&a.members, rng)?; + contact_records = self.prepare_roster_stubs(&a.members)?; rec.members = a .members .iter() @@ -1481,8 +1986,7 @@ impl Node { }); } } - self.events - .push_back(Event::GroupUpdated { group: a.group }); + group_changed = true; } rec } @@ -1490,10 +1994,11 @@ impl Node { // The sender-key half: honored from any current member. if !rec.members.iter().any(|m| m.peer == peer) { - return Ok(false); + return Ok((false, false)); } - let mut chains = match self.store.get_group_chain(&rec.id, &peer)? { - Some(blob) => decode_device_group_chains(&blob, peer)?, + let before_chain = self.store.get_group_chain(&rec.id, &peer)?; + let mut chains = match before_chain.as_ref() { + Some(blob) => decode_device_group_chains(blob, peer)?, None => Vec::new(), }; let replace = chains @@ -1502,7 +2007,7 @@ impl Node { // Same chain id: the stored state reads from an earlier (or // equal) iteration — strictly more capable, keep it. .is_none_or(|entry| entry.chain.key_id() != a.key_id); - if replace { + let after_chain = if replace { let chain = GroupReceiverChain::new(a.key_id, &a.chain_key, a.iteration); if let Some(entry) = chains.iter_mut().find(|entry| entry.device == peer_device) { entry.chain = chain; @@ -1512,29 +2017,106 @@ impl Node { chain, }); } - let encoded = encode_device_group_chains(&mut chains)?; - self.store.put_group_chain(&rec.id, &peer, &encoded, rng)?; + Some(encode_device_group_chains(&mut chains)?) + } else { + None + }; + + let group_transitions = (before_group.as_ref() != Some(&rec)) + .then_some(GroupStateTransition { + before: before_group.as_ref(), + after: Some(&rec), + }) + .into_iter() + .collect::>(); + let removed_peers = before_group + .as_ref() + .map(|before| { + let retained = rec + .members + .iter() + .map(|member| member.peer) + .collect::>(); + before + .members + .iter() + .filter(|member| !retained.contains(&member.peer)) + .map(|member| member.peer) + .collect::>() + }) + .unwrap_or_default(); + let removed_chain_rows = self + .store + .group_chains(&rec.id)? + .into_iter() + .filter(|(account, _)| removed_peers.contains(account)) + .collect::>(); + let mut chain_transitions = removed_chain_rows + .iter() + .map(|(account, chain)| GroupChainStateTransition { + group: rec.id, + peer: *account, + before: Some(chain.as_slice()), + after: None, + }) + .collect::>(); + if let Some(after_chain) = after_chain.as_ref() { + chain_transitions.push(GroupChainStateTransition { + group: rec.id, + peer, + before: before_chain.as_ref().map(|chain| chain.as_slice()), + after: Some(after_chain), + }); + } + let contact_transitions = contact_records + .iter() + .map(|contact| ContactTransition { + before: None, + after: contact, + }) + .collect::>(); + let mut events = contact_records + .iter() + .map(|contact| Event::ContactAdded { peer: contact.peer }) + .collect::>(); + if group_changed { + events.push(Event::GroupUpdated { group: a.group }); + } + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &group_transitions, + chains: &chain_transitions, + contacts: &contact_transitions, + authorities: &[], + delete_controls: core::slice::from_ref(control), + presentation_changed: !events.is_empty(), + }), + rng, + )?; + self.accept_commit_receipt(receipt, events); + if after_chain.is_some() { // Stashed group messages on this chain may decrypt now. *established = true; } - self.store.put_group(&rec, rng)?; - Ok(true) + Ok((true, true)) } fn apply_group_leave( &mut self, peer: [u8; 32], group: &[u8; 32], + control: &DeferredControlRecord, rng: &mut impl CryptoRngCore, - ) -> Result { - let Some(mut rec) = self.store.get_group(group)? else { - return Ok(true); // unknown group: terminal no-op + ) -> Result<(bool, bool)> { + let Some(before) = self.store.get_group(group)? else { + return Ok((true, false)); // unknown group: terminal no-op }; - if !rec.members.iter().any(|m| m.peer == peer) { - return Ok(true); + if !before.members.iter().any(|m| m.peer == peer) { + return Ok((true, false)); } + let mut rec = before.clone(); rec.members.retain(|m| m.peer != peer); - self.store.delete_group_chain(group, &peer)?; + let before_chain = self.store.get_group_chain(group, &peer)?; let me = self.identity.public().ed; if rec.creator == me { // Authority: re-key the shrunk roster so the leaver cannot even @@ -1545,27 +2127,49 @@ impl Node { } // Every remaining member rotates on membership shrink (spec §6). self.rotate_group(&mut rec, rng)?; - self.store.put_group(&rec, rng)?; - self.events.push_back(Event::GroupUpdated { group: *group }); - Ok(true) + let chain_transitions = before_chain + .as_ref() + .map(|chain| GroupChainStateTransition { + group: *group, + peer, + before: Some(chain.as_slice()), + after: None, + }) + .into_iter() + .collect::>(); + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[GroupStateTransition { + before: Some(&before), + after: Some(&rec), + }], + chains: &chain_transitions, + contacts: &[], + authorities: &[], + delete_controls: core::slice::from_ref(control), + presentation_changed: true, + }), + rng, + )?; + self.accept_commit_receipt(receipt, [Event::GroupUpdated { group: *group }]); + Ok((true, true)) } fn apply_group_remove_notice( &mut self, peer: [u8; 32], group: &[u8; 32], + control: &DeferredControlRecord, rng: &mut impl CryptoRngCore, - ) -> Result { - let _ = rng; + ) -> Result<(bool, bool)> { let Some(rec) = self.store.get_group(group)? else { - return Ok(true); + return Ok((true, false)); }; if rec.creator != peer { - return Ok(true); // only the creator removes; anything else is noise + return Ok((true, false)); // only the creator removes } - self.store.delete_group(group)?; // history stays - self.events.push_back(Event::GroupUpdated { group: *group }); - Ok(true) + self.commit_group_removal(&rec, control, rng)?; + Ok((true, true)) } // ---- receipts and delivery ladder ---------------------------------------- @@ -1618,6 +2222,34 @@ impl Node { // ---- internals ------------------------------------------------------- + fn candidate_group_seal( + &self, + chain: &mut GroupSenderChain, + header: &GroupHeaderKey, + group: &[u8; 32], + plaintext: &[u8], + rng: &mut impl CryptoRngCore, + ) -> Result { + let step = self.begin_crypto_step()?; + let message = chain.seal(header, group, plaintext, rng); + self.finish_crypto_step(step)?; + Ok(message) + } + + fn candidate_group_open( + &self, + chain: &mut GroupReceiverChain, + group: &[u8; 32], + message: &GroupMessage, + iteration: u32, + now: u64, + ) -> Result, kult_crypto::CryptoError>> { + let step = self.begin_crypto_step()?; + let plaintext = chain.open(group, message, iteration, now); + self.finish_crypto_step(step)?; + Ok(plaintext) + } + /// Fresh sending chain, everything reset: announces owed to the whole /// roster with the new snapshot. pub(crate) fn rotate_group( @@ -1633,41 +2265,76 @@ impl Node { Ok(()) } - /// Contact stubs for roster members we have never met: identity only, - /// no bundle, no hints — the DHT (or a later out-of-band exchange) - /// fills those in, exactly like a contact learned from an inbound - /// handshake. - pub(crate) fn adopt_roster_stubs( - &mut self, + pub(crate) fn prepare_roster_stubs( + &self, members: &[GroupMemberInfo], - rng: &mut impl CryptoRngCore, - ) -> Result<()> { + ) -> Result> { let me = self.identity.public().ed; - for m in members { - if m.peer == me || m.identity.is_empty() || self.store.get_contact(&m.peer)?.is_some() { + let mut contacts = Vec::new(); + for member in members { + if member.peer == me + || member.identity.is_empty() + || self.store.get_contact(&member.peer)?.is_some() + { continue; } - // Never store an identity blob that does not belong to the peer - // id it is filed under. - let Ok(identity) = postcard::from_bytes::(&m.identity) else { - continue; - }; - if identity.ed != m.peer { - continue; + let identity: IdentityPublic = + postcard::from_bytes(&member.identity).map_err(|_| NodeError::CorruptState)?; + if identity.ed != member.peer { + return Err(NodeError::CorruptState); } - self.store.put_contact( - &ContactRecord { - peer: m.peer, - identity: m.identity.clone(), - name: String::new(), - bundle: Vec::new(), - hints: Vec::new(), - verified: false, - }, - rng, - )?; - self.events.push_back(Event::ContactAdded { peer: m.peer }); + contacts.push(ContactRecord { + peer: member.peer, + identity: member.identity.clone(), + name: String::new(), + bundle: Vec::new(), + hints: Vec::new(), + verified: false, + }); } + Ok(contacts) + } + + pub(crate) fn commit_group_removal( + &mut self, + before: &GroupRecord, + control: &DeferredControlRecord, + rng: &mut impl CryptoRngCore, + ) -> Result<()> { + let chain_rows = self.store.group_chains(&before.id)?; + let chain_transitions = chain_rows + .iter() + .map(|(peer, chain)| GroupChainStateTransition { + group: before.id, + peer: *peer, + before: Some(chain.as_slice()), + after: None, + }) + .collect::>(); + let authority = self.store.get_group_authority(&before.id)?; + let authority_transitions = authority + .as_ref() + .map(|authority| GroupAuthorityStateTransition { + before: Some(authority), + after: None, + }) + .into_iter() + .collect::>(); + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[GroupStateTransition { + before: Some(before), + after: None, + }], + chains: &chain_transitions, + contacts: &[], + authorities: &authority_transitions, + delete_controls: core::slice::from_ref(control), + presentation_changed: true, + }), + rng, + )?; + self.accept_commit_receipt(receipt, [Event::GroupUpdated { group: before.id }]); Ok(()) } @@ -1675,16 +2342,15 @@ impl Node { /// exists (the delivery token needs it). `None` means "not yet" — the /// tick retries once the session appears. #[allow(clippy::too_many_arguments)] // exact fan-out identity, timing, and retention inputs - fn queue_group_copy( - &mut self, + fn prepare_group_copy( + &self, peer: &[u8; 32], wire: &[u8], group_msg_id: [u8; 16], class: QueueClass, retention_until: Option, now: u64, - rng: &mut impl CryptoRngCore, - ) -> Result> { + ) -> Result { let mut routes: Vec<[u8; 32]> = self .store .contact_devices_for(peer)? @@ -1696,21 +2362,25 @@ impl Node { } routes.sort_unstable(); routes.dedup(); + if routes.len() > kult_store::MAX_PAIRWISE_COMMIT_DEVICES { + return Err(NodeError::CorruptState); + } let existing = self.store.message_device_deliveries(&group_msg_id)?; - let mut first_wire = None; - let mut all_served = true; + let mut prepared = PreparedGroupCopy { + all_served: true, + ..PreparedGroupCopy::default() + }; for route in routes { - if let Some(wire_id) = existing + let existing_delivery = existing .iter() - .find(|delivery| delivery.account == *peer && delivery.device == route) - .and_then(|delivery| delivery.wire_id) - { - first_wire.get_or_insert(wire_id); + .find(|delivery| delivery.account == *peer && delivery.device == route); + if let Some(wire_id) = existing_delivery.and_then(|delivery| delivery.wire_id) { + prepared.first_wire.get_or_insert(wire_id); continue; } let Some(session) = self.sessions.get(&route) else { - all_served = false; + prepared.all_served = false; continue; }; let token = delivery_token( @@ -1728,32 +2398,31 @@ impl Node { None => Envelope::new(EnvelopeKind::GroupMessage, token, wire.to_vec()), }; let wire_id = envelope.content_id(); - first_wire.get_or_insert(wire_id); - self.store.put_message_device_delivery( - &MessageDeviceDeliveryRecord { - message: group_msg_id, - account: *peer, - device: route, - wire_id: Some(wire_id), - state: DeliveryState::Queued, - }, - rng, - )?; - self.store.queue_push( - &QueueItem { - peer: route, - msg_id: None, - group_msg_id: Some(group_msg_id), - class, - created_at: now, - attempts: 0, - next_attempt_at: now, - envelope, - }, - rng, - )?; + prepared.first_wire.get_or_insert(wire_id); + let after = MessageDeviceDeliveryRecord { + message: group_msg_id, + account: *peer, + device: route, + wire_id: Some(wire_id), + state: DeliveryState::Queued, + }; + if let Some(before) = existing_delivery { + prepared.delivery_updates.push((before.clone(), after)); + } else { + prepared.deliveries.push(after); + } + prepared.queue.push(QueueItem { + peer: route, + msg_id: None, + group_msg_id: Some(group_msg_id), + class, + created_at: now, + attempts: 0, + next_attempt_at: now, + envelope, + }); } - Ok(all_served.then_some(first_wire).flatten()) + Ok(prepared) } /// Encrypt one already-persisted Attachment manifest exactly once on the @@ -1767,53 +2436,119 @@ impl Node { now: u64, rng: &mut impl CryptoRngCore, ) -> Result { - let mut rec = self + let before_group = self .store .get_group(group)? .ok_or(NodeError::UnknownGroup)?; - let mut record = self + let mut after_group = before_group.clone(); + let before_record = self .store .group_messages(group)? .into_iter() .find(|record| record.direction == Direction::Outbound && &record.id == content_id) .ok_or(NodeError::UnknownAttachment)?; - if record + if before_record .deliveries .iter() .any(|delivery| delivery.wire_id.is_some()) { return Ok(false); } - for delivery in &record.deliveries { + for delivery in &before_record.deliveries { if !self.peer_has_live_device_sessions(&delivery.peer)? { return Ok(false); } } - if rec.sent_since_rotation >= GROUP_ROTATE_MSGS { - self.rotate_group(&mut rec, rng)?; + if after_group.sent_since_rotation >= GROUP_ROTATE_MSGS { + self.rotate_group(&mut after_group, rng)?; } - let mut chain = decode_chain(&rec.sender_chain)?; - let hk = GroupHeaderKey::derive(&rec.secret); - let wire = chain.seal(&hk, group, &pad(&record.body)?, rng).encode(); - rec.sender_chain = encode_chain(&chain)?; - rec.sent_since_rotation += 1; - for delivery in record.deliveries.iter_mut() { - delivery.wire_id = self.queue_group_copy( + let mut chain = decode_chain(&after_group.sender_chain)?; + let hk = GroupHeaderKey::derive(&after_group.secret); + let wire = self + .candidate_group_seal(&mut chain, &hk, group, &pad(&before_record.body)?, rng)? + .encode(); + after_group.sender_chain = encode_chain(&chain)?; + after_group.sent_since_rotation = after_group + .sent_since_rotation + .checked_add(1) + .ok_or(NodeError::CorruptState)?; + let mut after_record = before_record.clone(); + let mut queue = Vec::new(); + let mut delivery_rows = Vec::new(); + let mut delivery_pairs = Vec::new(); + for delivery in after_record.deliveries.iter_mut() { + let prepared = self.prepare_group_copy( &delivery.peer, &wire, - record.id, + after_record.id, QueueClass::Bulk, - ephemeral_retention(&record.body), + ephemeral_retention(&after_record.body), now, - rng, )?; - if delivery.wire_id.is_none() { + if !prepared.all_served || prepared.first_wire.is_none() { return Ok(false); } + delivery.wire_id = prepared.first_wire; + queue.extend(prepared.queue); + delivery_rows.extend(prepared.deliveries); + delivery_pairs.extend(prepared.delivery_updates); + } + after_record.wire_body = None; + let delivery_updates = delivery_pairs + .iter() + .map(|(before, after)| DeliveryTransition { before, after }) + .collect::>(); + let media_pairs = self + .store + .media_transfers()? + .into_iter() + .filter_map(|record| match record { + MediaRecord::Available(before) + if before.scope == MediaScope::Group + && before.direction == MediaDirection::Outbound + && before.scope_id == *group + && before.manifest_content_id == *content_id + && before.state == MediaTransferState::Queued => + { + let mut after = before.clone(); + after.state = MediaTransferState::Transferring; + after.updated_at = now; + Some((before, after)) + } + _ => None, + }) + .collect::>(); + let media_transfers = media_pairs + .iter() + .map(|(before, after)| MediaTransferTransition { before, after }) + .collect::>(); + let receipt = self.store.commit_plan( + CommitPlan::GroupSend(GroupSendPlan { + group: Some(GroupTransition { + before: &before_group, + after: &after_group, + }), + message: None, + message_update: Some(GroupMessageTransition { + before: &before_record, + after: &after_record, + }), + deliveries: &delivery_rows, + delivery_updates: &delivery_updates, + queue: &queue, + scheduled: None, + ephemeral: None, + media_transfers: &media_transfers, + delete_chains: &[], + authority: None, + presentation_changed: !media_transfers.is_empty(), + }), + rng, + )?; + self.accept_commit_receipt(receipt, []); + for (_, transfer) in media_pairs { + self.emit_attachment_update(&transfer.local_id)?; } - record.wire_body = None; - self.store.update_group_message(&record, rng)?; - self.store.put_group(&rec, rng)?; Ok(true) } @@ -1829,6 +2564,121 @@ impl Node { now: u64, rng: &mut impl CryptoRngCore, ) -> Result> { + let Some(prepared) = self.prepare_group_control(peer, payload, now, rng)? else { + return Ok(None); + }; + let preferred_wire = prepared.preferred_wire; + self.commit_prepared_group_control(prepared, None, &[], &[], rng)?; + Ok(Some(preferred_wire)) + } + + #[allow(clippy::too_many_arguments)] // encrypted response and exact authority consequence + pub(crate) fn queue_group_control_with_authority_response( + &mut self, + peer: &[u8; 32], + payload: &GroupControlPayload, + authority_before: &GroupAuthorityRecord, + authority_after: &GroupAuthorityRecord, + control: &DeferredControlRecord, + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result { + let Some(prepared) = self.prepare_group_control(peer, payload, now, rng)? else { + return Ok(false); + }; + self.commit_prepared_group_control( + prepared, + None, + &[GroupAuthorityStateTransition { + before: Some(authority_before), + after: Some(authority_after), + }], + core::slice::from_ref(control), + rng, + )?; + Ok(true) + } + + pub(crate) fn queue_group_control_with_control_ack( + &mut self, + peer: &[u8; 32], + payload: &GroupControlPayload, + control: &DeferredControlRecord, + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result { + let Some(prepared) = self.prepare_group_control(peer, payload, now, rng)? else { + return Ok(false); + }; + self.commit_prepared_group_control( + prepared, + None, + &[], + core::slice::from_ref(control), + rng, + )?; + Ok(true) + } + + #[allow(clippy::too_many_arguments)] // exact pending owner and pairwise crypto inputs + fn queue_group_announce( + &mut self, + peer: &[u8; 32], + payload: &GroupControlPayload, + before_group: &GroupRecord, + after_group: &mut GroupRecord, + pending_index: usize, + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result> { + let prepared = self.prepare_group_control(peer, payload, now, rng)?; + let preferred_wire = prepared.as_ref().map(|prepared| prepared.preferred_wire); + let pending = after_group + .pending + .get_mut(pending_index) + .filter(|pending| pending.peer == *peer) + .ok_or(NodeError::CorruptState)?; + pending.wire_id = preferred_wire; + pending.last_sent = now.max(pending.last_sent.saturating_add(1)); + + if let Some(prepared) = prepared { + self.commit_prepared_group_control( + prepared, + Some(GroupTransition { + before: before_group, + after: after_group, + }), + &[], + &[], + rng, + )?; + } else { + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[GroupStateTransition { + before: Some(before_group), + after: Some(after_group), + }], + chains: &[], + contacts: &[], + authorities: &[], + delete_controls: &[], + presentation_changed: false, + }), + rng, + )?; + self.accept_commit_receipt(receipt, []); + } + Ok(preferred_wire) + } + + fn prepare_group_control( + &mut self, + peer: &[u8; 32], + payload: &GroupControlPayload, + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result> { let Some(contact) = self.store.get_contact(peer)? else { return Ok(None); }; @@ -1855,11 +2705,14 @@ impl Node { .then_with(|| left.device.cmp(&right.device)) }); routes.dedup_by_key(|route| route.device); + if routes.len() > kult_store::MAX_PAIRWISE_COMMIT_DEVICES { + return Err(NodeError::CorruptState); + } let bytes = zeroize::Zeroizing::new(payload.encode()); let padded = pad(&bytes)?; let mut preferred_wire = None; - let mut prepared: Vec<([u8; 32], Option, Session, bool)> = Vec::new(); + let mut prepared = Vec::new(); let mut queue = Vec::new(); for route in routes { let device = route.device; @@ -1908,48 +2761,77 @@ impl Node { next_attempt_at: now, envelope, }); - prepared.push((device, before, after, reset)); + prepared.push(PreparedGroupControlRoute { + device, + before, + after, + reset, + }); } if prepared.is_empty() { return Ok(None); } + Ok(Some(PreparedGroupControl { + preferred_wire: preferred_wire.expect("prepared route has a control ciphertext"), + routes: prepared, + queue, + })) + } + + fn commit_prepared_group_control( + &mut self, + prepared: PreparedGroupControl, + group: Option>, + authorities: &[GroupAuthorityStateTransition<'_>], + delete_controls: &[DeferredControlRecord], + rng: &mut impl CryptoRngCore, + ) -> Result<()> { let transitions = prepared + .routes .iter() - .map(|(device, before, after, _)| SessionTransition { - peer_device: *device, - before: before.as_ref(), - after, + .map(|route| SessionTransition { + peer_device: route.device, + before: route.before.as_ref(), + after: &route.after, }) .collect::>(); let clear_capabilities = prepared + .routes .iter() - .filter_map(|(device, _, _, reset)| reset.then_some(*device)) + .filter_map(|route| route.reset.then_some(route.device)) .collect::>(); - self.store.commit_plan( + let groups = group.into_iter().collect::>(); + let receipt = self.store.commit_plan( CommitPlan::PairwiseSend(PairwiseSendPlan { sessions: &transitions, message: None, message_update: None, deliveries: &[], delivery_updates: &[], - queue: &queue, + queue: &prepared.queue, + groups: &groups, + authorities, scheduled: None, clear_capabilities: &clear_capabilities, clear_reset_markers: &[], ephemeral: None, + media_transfers: &[], + media_objects: &[], + delete_controls, presentation_changed: false, }), rng, )?; self.before_memory_replacement()?; - for (device, _, after, reset) in prepared { - self.sessions.insert(device, after); - if reset { - self.capabilities_advertised.remove(&device); + for route in prepared.routes { + self.sessions.insert(route.device, route.after); + if route.reset { + self.capabilities_advertised.remove(&route.device); } } self.after_memory_replacement()?; - Ok(preferred_wire) + self.accept_commit_receipt(receipt, []); + Ok(()) } /// Roster members met only through an announce have identity but no @@ -1964,21 +2846,36 @@ impl Node { if self.peer_has_session_or_bundle(peer)? || self.discoveries.is_empty() { return Ok(()); } - let Some(mut contact) = self.store.get_contact(peer)? else { + let Some(before) = self.store.get_contact(peer)? else { return Ok(()); }; - if !contact.bundle.is_empty() { + if !before.bundle.is_empty() { return Ok(()); } - let Ok(identity) = postcard::from_bytes::(&contact.identity) else { + let Ok(identity) = postcard::from_bytes::(&before.identity) else { return Ok(()); }; let Some(bundle) = self.lookup_bundle(identity.address_digest(), now).await else { return Ok(()); }; + let mut contact = before.clone(); contact.hints = bundle.relay_hints.clone(); contact.bundle = bundle.encode(); - self.store.put_contact(&contact, rng)?; + let receipt = self.store.commit_plan( + CommitPlan::GroupState(GroupStatePlan { + groups: &[], + chains: &[], + contacts: &[ContactTransition { + before: Some(&before), + after: &contact, + }], + authorities: &[], + delete_controls: &[], + presentation_changed: false, + }), + rng, + )?; + self.accept_commit_receipt(receipt, []); Ok(()) } } @@ -2007,6 +2904,22 @@ fn ephemeral_retention(body: &[u8]) -> Option { } } +fn valid_roster(members: &[GroupMemberInfo]) -> bool { + let peers = members + .iter() + .map(|member| member.peer) + .collect::>(); + !members.is_empty() + && members.len() <= MAX_GROUP_AUTHORITY_MEMBERS + && peers.len() == members.len() + && members.iter().all(|member| { + !member.identity.is_empty() + && member.identity.len() <= MAX_GROUP_MEMBER_IDENTITY_LEN + && postcard::from_bytes::(&member.identity) + .is_ok_and(|identity| identity.ed == member.peer) + }) +} + /// Announce entries for every roster member but `me`, snapshotting `chain` /// at its current state (the entitlement point). pub(crate) fn pending_for( diff --git a/crates/kult-node/src/lib.rs b/crates/kult-node/src/lib.rs index e5688a1..3b96d62 100644 --- a/crates/kult-node/src/lib.rs +++ b/crates/kult-node/src/lib.rs @@ -35,7 +35,7 @@ #[cfg(feature = "test-failpoints")] use std::cell::{Cell, RefCell}; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::sync::Arc; use std::time::Instant; @@ -59,17 +59,18 @@ use kult_protocol::{ MIN_EPHEMERAL_LIFETIME_SECS, REASSEMBLY_WINDOW_SECS, }; use kult_store::{ - CommitPlan, CommitReceipt, ContactDeviceDelete, ContactDeviceRecord, ContactRecord, - ConversationId, ConversationMetadata, DeferredControlKind, DeferredControlRecord, - DeliveryState, DeliveryTransition, DeviceStateRecord, Direction, EphemeralConversation, - EphemeralMode, EphemeralRecord, EphemeralState, EphemeralTransition, GroupMessageDelete, - GroupMessageRecord, GroupMessageTransition, GroupRecord, GroupTransition, HandshakeReceivePlan, - LocalMetadataKey, LocalMetadataRecord, MaintenancePlan, MediaDelete, MediaObjectRecord, - MediaTransferRecord, MessageDelete, MessageDeviceDeliveryRecord, MessageRecord, - MessageTransition, NoteMessageRecord, PairwiseReceivePlan, PairwiseSendPlan, PendingDelete, + AttachmentStatePlan, CommitPlan, CommitReceipt, ContactDeviceDelete, ContactDeviceRecord, + ContactRecord, ConversationId, ConversationMetadata, DeferredControlKind, + DeferredControlRecord, DeliveryState, DeliveryTransition, DeviceStateRecord, Direction, + EphemeralConversation, EphemeralMode, EphemeralRecord, EphemeralState, EphemeralTransition, + GroupMessageDelete, GroupMessageRecord, GroupMessageTransition, GroupRecord, GroupTransition, + HandshakeReceivePlan, LocalMetadataKey, LocalMetadataRecord, MaintenancePlan, MediaDelete, + MediaObjectRecord, MediaObjectTransition, MediaTransferRecord, MediaTransferTransition, + MessageDelete, MessageDeviceDeliveryRecord, MessageRecord, MessageTransition, + NoteMessageRecord, PairwiseReceivePlan, PairwiseSendPlan, PendingDelete, PrekeyPublishPlan, PrekeyTransition, QueueClass, QueueDelete, QueueItem, QueueTransition, ReceiptReceivePlan, ScheduledConversation as StoreScheduledConversation, ScheduledMessageRecord, SessionTransition, - Store, + Store, MAX_MAINTENANCE_TRANSITIONS, }; use kult_transport::{CostClass, DeliveryHint, Discovery, Reachability, Transport}; @@ -688,13 +689,26 @@ impl Node { profile: KdfProfile, rng: &mut impl CryptoRngCore, ) -> Result { - let store = Store::restore_backup(path, backup, mnemonic, passphrase, profile, rng)?; + let (store, vault) = Store::restore_backup_with_initializer( + path, + backup, + mnemonic, + passphrase, + profile, + rng, + |store, rng| { + let identity = store + .get_identity()? + .ok_or(kult_store::StoreError::NotAStore)?; + if store.get_device_state()?.is_none() { + devices::initialize_fresh_device(store, &identity, rng)?; + } + let vault = PrekeyVault::generate(rng); + store.put_prekeys(&vault.encode(), rng)?; + Ok(vault) + }, + )?; let identity = store.get_identity()?.ok_or(NodeError::CorruptState)?; - if store.get_device_state()?.is_none() { - devices::initialize_fresh_device(&store, &identity, rng)?; - } - let vault = PrekeyVault::generate(rng); - store.put_prekeys(&vault.encode(), rng)?; Self::assemble(store, identity, vault) } @@ -871,8 +885,11 @@ impl Node { now: u64, rng: &mut impl CryptoRngCore, ) -> Result> { + let before_vault = self.vault.encode(); let mut candidate_vault = self.vault.clone(); + let step = self.begin_crypto_step()?; let opk = candidate_vault.fresh_opk(rng); + self.finish_crypto_step(step)?; let linked = self.device_state.manifest.devices.len() > 1; let signing_identity = if linked { &self.device_identity @@ -897,7 +914,16 @@ impl Node { } else { bundle.encode() }; - self.store.put_prekeys(&candidate_vault.encode(), rng)?; + let after_vault = candidate_vault.encode(); + self.store.commit_plan( + CommitPlan::PrekeyPublish(PrekeyPublishPlan { + prekeys: PrekeyTransition { + before: &before_vault, + after: &after_vault, + }, + }), + rng, + )?; self.before_memory_replacement()?; self.vault = candidate_vault; self.own_hints = hints.to_vec(); @@ -1903,10 +1929,15 @@ impl Node { deliveries: &deliveries, delivery_updates: &[], queue: &queue, + groups: &[], + authorities: &[], scheduled, clear_capabilities: &clear_capabilities, clear_reset_markers: &clear_reset_markers, ephemeral: ephemeral.as_ref(), + media_transfers: &[], + media_objects: &[], + delete_controls: &[], presentation_changed: true, }), rng, @@ -2166,8 +2197,43 @@ impl Node { self.device_state_dirty = false; } if !self.media_reconciled { - self.store.reconcile_media(rng)?; - self.media_reconciled = true; + let batch = self + .store + .prepare_media_reconciliation(MAX_MAINTENANCE_TRANSITIONS)?; + if batch.transitions.is_empty() { + self.store + .finish_media_reconciliation(batch.unknown_records)?; + self.media_reconciled = batch.complete; + } else { + let object_transitions = batch + .transitions + .iter() + .map(|(before, after)| MediaObjectTransition { before, after }) + .collect::>(); + let receipt = self.store.commit_plan( + CommitPlan::AttachmentState(AttachmentStatePlan { + media_transfers: &[], + media_objects: &object_transitions, + delete_controls: &[], + presentation_changed: true, + }), + rng, + )?; + let transfers = batch + .transitions + .iter() + .map(|(_, after)| after.transfer_id) + .collect::>(); + self.accept_commit_receipt(receipt, []); + for transfer in transfers { + self.emit_attachment_update(&transfer)?; + } + if batch.complete { + self.store + .finish_media_reconciliation(batch.unknown_records)?; + self.media_reconciled = true; + } + } } self.apply_deferred_controls(now, rng)?; // Expiry is core-owned and runs before any queue activation, receive, @@ -2377,7 +2443,6 @@ impl Node { // established (a handshake later in the batch can unlock messages // earlier in it). Each pass consumes at least one envelope, so // this terminates. - let mut acks: Vec<([u8; 32], [u8; 16])> = Vec::new(); let mut pending_acks: Vec = Vec::new(); loop { let mut stash = Vec::new(); @@ -2391,7 +2456,6 @@ impl Node { }, now, rng, - &mut acks, &mut established, )? { Consumed::Done => { @@ -2455,17 +2519,10 @@ impl Node { // only under a fresh F4 bulk-capable verdict. self.activate_attachment_transfers(now, rng).await?; - // 3. Acknowledge consumed messages with end-to-end encrypted - // receipts, and NACK the missing fragment indices of stale + // 3. NACK the missing fragment indices of stale // partials (selective retransmission, docs/05-transports.md §4.2 - // rule 2) — batched per peer, acks and nacks in one envelope. - for (peer, content_id) in &acks { - self.store.put_receipt_replay(content_id, peer, now, rng)?; - } - let mut acks_by_peer: BTreeMap<[u8; 32], Vec<[u8; 16]>> = BTreeMap::new(); - for (peer, content_id) in acks { - acks_by_peer.entry(peer).or_default().push(content_id); - } + // rule 2). Accepted messages already own their encrypted receipt + // and duplicate-replay route inside the receive commit plan. let mut nacks_by_peer: BTreeMap<[u8; 32], FragNacks> = BTreeMap::new(); for (id, missing) in self.stale_partials(now) { // The fragment's delivery token names the session to ask. @@ -2482,15 +2539,9 @@ impl Node { } nacks_by_peer.entry(peer).or_default().push((id, missing)); } - let receipt_peers: BTreeSet<[u8; 32]> = acks_by_peer - .keys() - .chain(nacks_by_peer.keys()) - .copied() - .collect(); - for peer in receipt_peers { - let acks = acks_by_peer.remove(&peer).unwrap_or_default(); + for peer in nacks_by_peer.keys().copied().collect::>() { let nacks = nacks_by_peer.remove(&peer).unwrap_or_default(); - self.queue_receipt(&peer, acks, nacks, now, rng)?; + self.queue_receipt(&peer, Vec::new(), nacks, now, rng)?; } if !pending_acks.is_empty() { self.store.commit_plan( @@ -2986,25 +3037,14 @@ impl Node { Ok(true) } StoreScheduledConversation::Group(group) => { - self.group_send_with_id( - &group, - &scheduled.body, - scheduled.id, - scheduled.not_before, - now, - rng, - )?; - Ok(false) + self.activate_scheduled_group_message(&group, &scheduled, now, rng)?; + Ok(true) } } })(); match activation { Ok(true) => {} - Ok(false) => { - self.store.delete_scheduled_message(&scheduled.id)?; - self.events - .push_back(Event::ScheduledMessageActivated { id: scheduled.id }); - } + Ok(false) => unreachable!("scheduled activation reports committed success"), Err(_) => continue, } } @@ -3134,10 +3174,15 @@ impl Node { deliveries: &[], delivery_updates: &[], queue: &queue, + groups: &[], + authorities: &[], scheduled: None, clear_capabilities: &clear_capabilities, clear_reset_markers: &[marker], ephemeral: None, + media_transfers: &[], + media_objects: &[], + delete_controls: &[], presentation_changed: false, }), rng, @@ -3354,10 +3399,15 @@ impl Node { deliveries: &[], delivery_updates: &delivery_updates, queue: &queue, + groups: &[], + authorities: &[], scheduled: None, clear_capabilities: &clear_capabilities, clear_reset_markers: &[], ephemeral: None, + media_transfers: &[], + media_objects: &[], + delete_controls: &[], presentation_changed: false, }), rng, @@ -3383,33 +3433,30 @@ impl Node { .store .deferred_controls(MAX_DEFERRED_CONTROLS_PER_TICK)? { - let applied = match control.kind { + let (applied, acknowledged) = match control.kind { DeferredControlKind::AttachmentBulk => { - self.apply_attachment_bulk( + let acknowledged = self.apply_attachment_bulk( control.peer, control.peer_device, &control.body, now, + &control, rng, )?; - true + (true, acknowledged) } DeferredControlKind::GroupControl => { let mut established = false; - self.apply_group_control( - control.peer, - control.peer_device, - &control.body, - now, - rng, - &mut established, - )? + self.apply_group_control(&control, now, rng, &mut established)? } }; if !applied { continue; } made_progress = true; + if acknowledged { + continue; + } let receipt = self.store.commit_plan( CommitPlan::Maintenance(MaintenancePlan { seen: &[], @@ -3517,7 +3564,6 @@ impl Node { origin: ConsumeOrigin, now: u64, rng: &mut impl CryptoRngCore, - acks: &mut Vec<([u8; 32], [u8; 16])>, established: &mut bool, ) -> Result { // Multipath duplicates of anything already consumed are dropped here. @@ -3570,8 +3616,11 @@ impl Node { // Fragments never nest (we only fragment whole envelopes); // treat nested ones as malformed. if origin.depth > 0 { - self.store.mark_seen(&env.content_id())?; - return Ok(Consumed::Done); + return self.commit_terminal_input( + env.content_id(), + origin.pending_sequence, + rng, + ); } // Remember which delivery token this partial rides under so // the NACK for its missing pieces (selective retransmission, @@ -3602,7 +3651,6 @@ impl Node { }, now, rng, - acks, established, )? { // Reassembled before its session exists — stash @@ -3617,12 +3665,14 @@ impl Node { Ok(Consumed::Done) } EnvelopeKind::Handshake => { - self.consume_handshake(env, origin.pending_sequence, now, rng, acks, established) + self.consume_handshake(env, origin.pending_sequence, now, rng, established) } EnvelopeKind::Message | EnvelopeKind::Receipt | EnvelopeKind::GroupControl => { - self.consume_ratchet(env, origin.pending_sequence, now, rng, acks, established) + self.consume_ratchet(env, origin.pending_sequence, now, rng, established) + } + EnvelopeKind::GroupMessage => { + self.consume_group_message(env, origin.pending_sequence, now, rng) } - EnvelopeKind::GroupMessage => self.consume_group_message(env, now, rng, acks), } } @@ -3632,7 +3682,6 @@ impl Node { pending_sequence: Option, now: u64, rng: &mut impl CryptoRngCore, - _acks: &mut [([u8; 32], [u8; 16])], established: &mut bool, ) -> Result { // Every failure below is permanent for this envelope (it cannot @@ -3990,7 +4039,6 @@ impl Node { pending_sequence: Option, now: u64, rng: &mut impl CryptoRngCore, - _acks: &mut [([u8; 32], [u8; 16])], _established: &mut bool, ) -> Result { let Some(peer_device) = self.match_session(&env.token, now) else { @@ -4743,15 +4791,74 @@ impl Node { retention_until: Option, now: u64, rng: &mut impl CryptoRngCore, + ) -> Result> { + self.commit_pairwise_envelopes_with_effects( + devices, + padded, + kind, + class, + retention_until, + &[], + &[], + &[], + now, + rng, + ) + } + + #[allow(clippy::too_many_arguments)] // exact crypto and durable consequence boundary + pub(crate) fn commit_pairwise_envelopes_with_effects( + &mut self, + devices: &[[u8; 32]], + padded: &[u8], + kind: EnvelopeKind, + class: QueueClass, + retention_until: Option, + media_transfers: &[MediaTransferTransition<'_>], + media_objects: &[MediaObjectTransition<'_>], + delete_controls: &[DeferredControlRecord], + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result> { + self.commit_pairwise_payloads_with_effects( + devices, + &[padded], + kind, + class, + retention_until, + media_transfers, + media_objects, + delete_controls, + now, + rng, + ) + } + + #[allow(clippy::too_many_arguments)] // exact crypto and durable consequence boundary + pub(crate) fn commit_pairwise_payloads_with_effects( + &mut self, + devices: &[[u8; 32]], + padded_payloads: &[&[u8]], + kind: EnvelopeKind, + class: QueueClass, + retention_until: Option, + media_transfers: &[MediaTransferTransition<'_>], + media_objects: &[MediaObjectTransition<'_>], + delete_controls: &[DeferredControlRecord], + now: u64, + rng: &mut impl CryptoRngCore, ) -> Result> { let mut routes = devices.to_vec(); routes.sort_unstable(); routes.dedup(); - if routes.is_empty() || routes.len() > kult_store::MAX_PAIRWISE_COMMIT_DEVICES { + if routes.is_empty() + || routes.len() > kult_store::MAX_PAIRWISE_COMMIT_DEVICES + || padded_payloads.is_empty() + { return Err(NodeError::NoSession); } let mut prepared = Vec::with_capacity(routes.len()); - let mut queue = Vec::with_capacity(routes.len()); + let mut queue = Vec::with_capacity(routes.len().saturating_mul(padded_payloads.len())); for device in routes { let before = self .sessions @@ -4759,31 +4866,37 @@ impl Node { .cloned() .ok_or(NodeError::NoSession)?; let mut after = before.clone(); - let message = self.candidate_encrypt(&mut after, rng, now, padded)?; - let token = delivery_token( - &MailboxKey::from_bytes(*after.mailbox_key()), - epoch_day(now), - &device, - ); - let envelope = match retention_until { - Some(deadline) => Envelope::new_retained(kind, token, deadline, message.encode())?, - None => Envelope::new(kind, token, message.encode()), - }; - queue.push(QueueItem { - peer: device, - msg_id: None, - group_msg_id: None, - class, - created_at: now, - attempts: 0, - next_attempt_at: now, - envelope: envelope.clone(), - }); + let mut final_envelope = None; + for padded in padded_payloads { + let message = self.candidate_encrypt(&mut after, rng, now, padded)?; + let token = delivery_token( + &MailboxKey::from_bytes(*after.mailbox_key()), + epoch_day(now), + &device, + ); + let envelope = match retention_until { + Some(deadline) => { + Envelope::new_retained(kind, token, deadline, message.encode())? + } + None => Envelope::new(kind, token, message.encode()), + }; + queue.push(QueueItem { + peer: device, + msg_id: None, + group_msg_id: None, + class, + created_at: now, + attempts: 0, + next_attempt_at: now, + envelope: envelope.clone(), + }); + final_envelope = Some(envelope); + } prepared.push(PreparedPairwiseRoute { route: device, before: Some(before), after, - envelope, + envelope: final_envelope.expect("payloads are non-empty"), resets_capabilities: false, }); } @@ -4803,11 +4916,16 @@ impl Node { deliveries: &[], delivery_updates: &[], queue: &queue, + groups: &[], + authorities: &[], scheduled: None, clear_capabilities: &[], clear_reset_markers: &[], ephemeral: None, - presentation_changed: false, + media_transfers, + media_objects, + delete_controls, + presentation_changed: !media_transfers.is_empty() || !media_objects.is_empty(), }), rng, )?; diff --git a/crates/kult-node/src/polls.rs b/crates/kult-node/src/polls.rs index 23155fa..a707428 100644 --- a/crates/kult-node/src/polls.rs +++ b/crates/kult-node/src/polls.rs @@ -258,6 +258,18 @@ impl Node { Some(GroupRole::Owner) if authority.owner == me => {} _ => return Err(NodeError::NotGroupOwner), } + self.advance_authority(group, now, rng)?; + self.commit_group_poll_moderated_close(group, poll_author, poll_id, now, rng) + } + + pub(crate) fn commit_group_poll_moderated_close( + &mut self, + group: &[u8; 32], + poll_author: [u8; 32], + poll_id: [u8; 16], + now: u64, + rng: &mut impl CryptoRngCore, + ) -> Result<[u8; 16]> { let poll = self .group_polls(group)? .into_iter() @@ -276,7 +288,6 @@ impl Node { revision: vote.revision, }) .collect::>(); - self.advance_authority(group, now, rng)?; let generation = self.group_authority(group)?.generation; let records = self.store.group_messages(group)?; let id = mint_unique_id(rng, |candidate| { diff --git a/crates/kult-node/tests/groups_e2e.rs b/crates/kult-node/tests/groups_e2e.rs index 9b74d7b..342f9db 100644 --- a/crates/kult-node/tests/groups_e2e.rs +++ b/crates/kult-node/tests/groups_e2e.rs @@ -316,6 +316,82 @@ async fn group_round_trip_receipts_and_restart() { )); } +#[tokio::test] +async fn partial_carrier_handoff_survives_sender_restart() { + let mut rng = StdRng::seed_from_u64(18); + let dir = tempfile::tempdir().unwrap(); + let net: Net = Arc::new(Mutex::new(HashMap::new())); + let (mut alice, mut bob, mut carol, _a_id, b_id, c_id) = trio(dir.path(), &net, &mut rng).await; + let gid = alice + .create_group("partial handoff", &[b_id, c_id], &mut rng) + .unwrap(); + settle_trio(&mut alice, &mut bob, &mut carol, &mut rng, NOW + 1).await; + + let id = alice + .group_send(&gid, b"survives one lost carrier copy", NOW + 30, &mut rng) + .unwrap(); + alice.tick(NOW + 31, &mut rng).await.unwrap(); + let removed = { + let mut network = net.lock().unwrap(); + let carol_inbox = network.entry(3).or_default(); + let before = carol_inbox.len(); + carol_inbox.retain(|envelope| envelope.kind != EnvelopeKind::GroupMessage); + before - carol_inbox.len() + }; + assert_eq!(removed, 1); + + bob.tick(NOW + 32, &mut rng).await.unwrap(); + alice.tick(NOW + 33, &mut rng).await.unwrap(); + let record = alice + .group_messages(&gid) + .unwrap() + .into_iter() + .find(|message| message.id == id) + .unwrap(); + assert_eq!( + record + .deliveries + .iter() + .find(|delivery| delivery.peer == b_id) + .unwrap() + .state, + DeliveryState::Delivered + ); + assert_eq!( + record + .deliveries + .iter() + .find(|delivery| delivery.peer == c_id) + .unwrap() + .state, + DeliveryState::Sent + ); + + let alice_path = dir.path().join("a.db"); + drop(alice); + let mut alice = Node::open(&alice_path, b"a").unwrap(); + alice.add_transport(Arc::new(MockLink { + net: net.clone(), + me: 1, + })); + alice.tick(NOW + 1_000, &mut rng).await.unwrap(); + assert_eq!( + group_bodies(&carol.tick(NOW + 1_001, &mut rng).await.unwrap()), + vec![b"survives one lost carrier copy".to_vec()] + ); + alice.tick(NOW + 1_002, &mut rng).await.unwrap(); + let record = alice + .group_messages(&gid) + .unwrap() + .into_iter() + .find(|message| message.id == id) + .unwrap(); + assert!(record + .deliveries + .iter() + .all(|delivery| delivery.state == DeliveryState::Delivered)); +} + // --------------------------------------------------------------------------- // 2. Membership: add mid-flight (no history for the newcomer), creator // removal (removed member is cut off and told, survivors rotate and keep diff --git a/crates/kult-store/src/backup.rs b/crates/kult-store/src/backup.rs index 5fe0731..5e69d9e 100644 --- a/crates/kult-store/src/backup.rs +++ b/crates/kult-store/src/backup.rs @@ -346,6 +346,33 @@ impl Store { profile: KdfProfile, rng: &mut impl CryptoRngCore, ) -> Result { + Self::restore_backup_with_initializer( + path, + backup, + mnemonic, + passphrase, + profile, + rng, + |_store, _rng| Ok(()), + ) + .map(|(store, ())| store) + } + + /// Restore and finish caller-owned state inside the sibling database + /// before its single atomic replacement makes the destination visible. + pub fn restore_backup_with_initializer( + path: &std::path::Path, + backup: &[u8], + mnemonic: &str, + passphrase: &[u8], + profile: KdfProfile, + rng: &mut R, + initializer: F, + ) -> Result<(Self, T)> + where + R: CryptoRngCore, + F: FnOnce(&Store, &mut R) -> Result, + { if backup.len() <= HEADER_LEN || u64::try_from(backup.len()).map_or(true, |length| length > MAX_BACKUP_FILE_BYTES) { @@ -517,6 +544,14 @@ impl Store { return Err(error); } }; + let initialized = match initializer(&store, rng) { + Ok(initialized) => initialized, + Err(error) => { + drop(store); + cleanup_restore_temporary(&temporary)?; + return Err(error); + } + }; store.validate_open_state()?; migration::sync_database_for_replacement(&store.conn)?; drop(store); @@ -534,7 +569,7 @@ impl Store { let database_lock = acquire_database_identity_lock(path)?; let store = Store::open_v2_with_parts(path, passphrase, conn, database_lock, lock, false)?; migration::cleanup_obsolete_siblings(&temporary)?; - Ok(store) + Ok((store, initialized)) } /// The Argon2id profile this store was created with. @@ -1010,34 +1045,85 @@ mod tests { set_restore_failpoint(phase); assert!(matches!( - Store::restore_backup( + Store::restore_backup_with_initializer( &target_path, &backup, &mnemonic, b"target-pass", TEST_KDF, &mut rng, + |store, rng| { + store.put_prekeys(&[phase; 4], rng)?; + Ok(()) + }, ), Err(StoreError::MigrationValidation) )); let restored = if phase == 1 { - Store::restore_backup( + Store::restore_backup_with_initializer( &target_path, &backup, &mnemonic, b"target-pass", TEST_KDF, &mut rng, + |store, rng| { + store.put_prekeys(&[phase; 4], rng)?; + Ok(()) + }, ) .unwrap() + .0 } else { Store::open(&target_path, b"target-pass").unwrap() }; assert_eq!(restored.get_identity().unwrap().unwrap().public(), expected); + assert_eq!( + restored.get_prekeys().unwrap().unwrap().as_slice(), + &[phase; 4] + ); assert!(!restore_temporary_path(&target_path).unwrap().exists()); } } + #[test] + fn restore_initializer_failure_leaves_no_visible_destination() { + let directory = tempfile::tempdir().unwrap(); + let source_path = directory.path().join("source.db"); + let target_path = directory.path().join("target.db"); + let mut rng = StdRng::seed_from_u64(0x7005); + let source = Store::create(&source_path, b"source-pass", TEST_KDF, &mut rng).unwrap(); + source + .put_identity(&Identity::generate(&mut rng), &mut rng) + .unwrap(); + let (backup, mnemonic) = source.export_backup(123, &mut rng).unwrap(); + drop(source); + + assert!(matches!( + Store::restore_backup_with_initializer( + &target_path, + &backup, + &mnemonic, + b"target-pass", + TEST_KDF, + &mut rng, + |_store, _rng| Err::<(), _>(StoreError::MigrationValidation), + ), + Err(StoreError::MigrationValidation) + )); + assert!(!target_path.exists()); + assert!(!restore_temporary_path(&target_path).unwrap().exists()); + Store::restore_backup( + &target_path, + &backup, + &mnemonic, + b"target-pass", + TEST_KDF, + &mut rng, + ) + .unwrap(); + } + #[test] fn backup_plaintext_contains_logical_records_not_store_artifacts() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/kult-store/src/commit.rs b/crates/kult-store/src/commit.rs index a96b7a8..84302d4 100644 --- a/crates/kult-store/src/commit.rs +++ b/crates/kult-store/src/commit.rs @@ -1,19 +1,22 @@ //! Bounded typed transactions for protocol-state transitions (ADR-0028). -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use rand_core::CryptoRngCore; use serde::{Deserialize, Serialize}; use zeroize::Zeroizing; use kult_crypto::Session; -use kult_protocol::{CapabilityControl, Envelope, EnvelopeKind}; +use kult_protocol::{ + CapabilityControl, Envelope, EnvelopeKind, MAX_GROUP_ADMIN_REQUESTS, + MAX_GROUP_AUTHORITY_MEMBERS, MAX_GROUP_MEMBER_IDENTITY_LEN, MAX_GROUP_NAME_LEN, +}; use crate::{ decode_exact, store_v2, ContactDeviceRecord, ContactRecord, DeliveryState, Direction, - EphemeralRecord, EphemeralState, GroupMessageRecord, GroupRecord, MediaObjectRecord, - MediaTransferRecord, MessageDeviceDeliveryRecord, MessageRecord, QueueItem, Result, - ScheduledMessageRecord, Store, StoreError, + EphemeralRecord, EphemeralState, GroupAuthorityRecord, GroupMessageRecord, GroupRecord, + MediaObjectRecord, MediaRecord, MediaTransferRecord, MessageDeviceDeliveryRecord, + MessageRecord, QueueItem, Result, ScheduledMessageRecord, Store, StoreError, }; /// Maximum logical durable mutations accepted by one typed commit plan. @@ -22,6 +25,14 @@ pub const MAX_COMMIT_MUTATIONS: usize = 512; pub const MAX_PAIRWISE_COMMIT_DEVICES: usize = 8; /// Maximum queue rows created by one protocol transition. pub const MAX_COMMIT_QUEUE_ROWS: usize = 128; +/// Maximum queue rows in a stable-v1 group fan-out (64 accounts × 8 devices). +pub const MAX_GROUP_COMMIT_QUEUE_ROWS: usize = 512; +/// Maximum durable mutations in one bounded group fan-out transaction. +pub const MAX_GROUP_COMMIT_MUTATIONS: usize = 2_048; +/// Maximum rows created while staging one bounded attachment manifest. +pub const MAX_ATTACHMENT_STAGE_MUTATIONS: usize = 256; +/// Maximum exact rows changed by one group roster or authority transition. +pub const MAX_GROUP_STATE_MUTATIONS: usize = 256; /// Maximum exact maintenance transitions applied in one transaction. pub const MAX_MAINTENANCE_TRANSITIONS: usize = 256; /// Maximum authenticated control records retained for post-commit work. @@ -135,6 +146,62 @@ pub struct GroupMessageTransition<'a> { pub after: &'a GroupMessageRecord, } +/// An exact sealed group receiver-chain replacement. +pub struct GroupChainTransition<'a> { + /// Exact group conversation. + pub group: [u8; 32], + /// Account whose sender chain this receiver state follows. + pub peer: [u8; 32], + /// Exact durable chain expected before the transaction. + pub before: &'a [u8], + /// Candidate chain produced without mutating durable or live state. + pub after: &'a [u8], +} + +/// A new or exact replacement signed group-authority record. +pub struct GroupAuthorityTransition<'a> { + /// Existing authority state, or `None` for the first signed generation. + pub before: Option<&'a GroupAuthorityRecord>, + /// Resulting signed authority state. + pub after: &'a GroupAuthorityRecord, +} + +/// New, exact replacement, or exact removal of one signed group-authority record. +pub struct GroupAuthorityStateTransition<'a> { + /// Existing authority state, or `None` when creating it. + pub before: Option<&'a GroupAuthorityRecord>, + /// Resulting authority state, or `None` when removing it with the group. + pub after: Option<&'a GroupAuthorityRecord>, +} + +/// New, exact replacement, or exact removal of one group record. +pub struct GroupStateTransition<'a> { + /// Existing group, or `None` when creating it. + pub before: Option<&'a GroupRecord>, + /// Resulting group, or `None` when removing it. + pub after: Option<&'a GroupRecord>, +} + +/// New, exact replacement, or exact removal of one group receiver-chain blob. +pub struct GroupChainStateTransition<'a> { + /// Group whose receiver-chain map changes. + pub group: [u8; 32], + /// Account whose device-chain aggregate changes. + pub peer: [u8; 32], + /// Exact existing blob, or `None` when creating it. + pub before: Option<&'a [u8]>, + /// Resulting blob, or `None` when removing it. + pub after: Option<&'a [u8]>, +} + +/// New or exact replacement contact stub adopted from authenticated group state. +pub struct ContactTransition<'a> { + /// Existing contact, or `None` for a newly authenticated roster stub. + pub before: Option<&'a ContactRecord>, + /// Resulting contact row. + pub after: &'a ContactRecord, +} + /// An exact ephemeral marker update. pub struct EphemeralTransition<'a> { /// Existing marker, or `None` when the transition creates it. @@ -151,6 +218,22 @@ pub struct MediaDelete<'a> { pub object_ids: &'a [[u8; 16]], } +/// An exact attachment-transfer metadata replacement. +pub struct MediaTransferTransition<'a> { + /// Exact durable value expected before the transaction. + pub before: &'a MediaTransferRecord, + /// Replacement value with the same local transfer identity. + pub after: &'a MediaTransferRecord, +} + +/// An exact attachment-object metadata replacement. +pub struct MediaObjectTransition<'a> { + /// Exact durable value expected before the transaction. + pub before: &'a MediaObjectRecord, + /// Replacement value with the same local object identity. + pub after: &'a MediaObjectRecord, +} + /// One exact contact endpoint removed by an authenticated manifest transition. pub struct ContactDeviceDelete<'a> { /// Exact durable endpoint expected before deletion. @@ -171,6 +254,10 @@ pub struct PairwiseSendPlan<'a> { pub delivery_updates: &'a [DeliveryTransition<'a>], /// Every ciphertext produced by the candidate sessions. pub queue: &'a [QueueItem], + /// Exact pending-announcement ownership updates for group controls. + pub groups: &'a [GroupTransition<'a>], + /// Exact authority bookkeeping owned by an encrypted group-control response. + pub authorities: &'a [GroupAuthorityStateTransition<'a>], /// Exact scheduled record consumed by activation, when applicable. pub scheduled: Option<&'a ScheduledMessageRecord>, /// Session-bound capability snapshots invalidated by new handshakes. @@ -179,6 +266,115 @@ pub struct PairwiseSendPlan<'a> { pub clear_reset_markers: &'a [[u8; 32]], /// Optional local ephemeral marker created with history. pub ephemeral: Option<&'a EphemeralRecord>, + /// Attachment transfer rows activated by this exact manifest send. + pub media_transfers: &'a [MediaTransferTransition<'a>], + /// Attachment object progress owned by an encrypted bulk control. + pub media_objects: &'a [MediaObjectTransition<'a>], + /// Accepted controls consumed by the resulting encrypted response. + pub delete_controls: &'a [DeferredControlRecord], + /// Whether presentation must be recoverable after commit. + pub presentation_changed: bool, +} + +/// Complete durable consequences of one sender-key group send or late fan-out. +pub struct GroupSendPlan<'a> { + /// Sender-chain/group-state replacement when this transition encrypts. + /// Late fan-out of an already retained ciphertext leaves this `None`. + pub group: Option>, + /// Immutable outbound group history created by this send. + pub message: Option<&'a GroupMessageRecord>, + /// Exact retained-history replacement for attachment activation or late fan-out. + pub message_update: Option>, + /// Honest per-device delivery rows created with ciphertext copies. + pub deliveries: &'a [MessageDeviceDeliveryRecord], + /// Exact placeholder or delivery-row replacements. + pub delivery_updates: &'a [DeliveryTransition<'a>], + /// Every recipient-scoped envelope that owns the produced group ciphertext. + pub queue: &'a [QueueItem], + /// Exact scheduled record consumed by activation, when applicable. + pub scheduled: Option<&'a ScheduledMessageRecord>, + /// Optional local ephemeral marker created with history. + pub ephemeral: Option<&'a EphemeralRecord>, + /// Attachment transfer rows activated by this exact manifest send. + pub media_transfers: &'a [MediaTransferTransition<'a>], + /// Receiver chains removed by the roster transition carried by this send. + pub delete_chains: &'a [GroupChainStateTransition<'a>], + /// Optional signed authority state committed with its immutable announcement. + pub authority: Option>, + /// Whether presentation must be recoverable after commit. + pub presentation_changed: bool, +} + +/// Complete durable consequences of one sender-key group receive. +pub struct GroupReceivePlan<'a> { + /// Detached group receiver-chain candidate. + pub chain: GroupChainTransition<'a>, + /// Detached pairwise sending-session candidate used by the receipt. + pub receipt_session: SessionTransition<'a>, + /// Accepted immutable group history, if this content creates one. + pub message: Option<&'a GroupMessageRecord>, + /// Optional local ephemeral marker or tombstone. + pub ephemeral: Option<&'a EphemeralRecord>, + /// Attachment-offer transfer metadata created by accepted content. + pub media_transfers: &'a [MediaTransferRecord], + /// Attachment-offer object metadata created by accepted content. + pub media_objects: &'a [MediaObjectRecord], + /// Encrypted receipt produced by the detached pairwise candidate. + pub queue: &'a [QueueItem], + /// Authenticated group envelope content id used for replay absorption. + pub content_id: [u8; 16], + /// Local receive time retained with duplicate-receipt routing. + pub received_at: u64, + /// Exact deferred source row, when this input came from the durable inbox. + pub source_pending: Option, + /// Whether presentation must be recoverable after commit. + pub presentation_changed: bool, +} + +/// Complete local metadata stage for one outbound attachment manifest. +/// +/// Chunk files are committed through the media store's file-first chunk +/// protocol after this plan succeeds. The manifest is not eligible for +/// encryption until every staged object is complete. +pub struct AttachmentStagePlan<'a> { + /// Pairwise history created by this stage. + pub message: Option<&'a MessageRecord>, + /// Group history created by this stage. + pub group_message: Option<&'a GroupMessageRecord>, + /// New transfer rows owned by the manifest history. + pub media_transfers: &'a [MediaTransferRecord], + /// New object rows owned by those transfers. + pub media_objects: &'a [MediaObjectRecord], + /// Optional view-once marker created with the manifest. + pub ephemeral: Option<&'a EphemeralRecord>, + /// Whether presentation must be recoverable after commit. + pub presentation_changed: bool, +} + +/// One bounded attachment lifecycle or progress transition. +pub struct AttachmentStatePlan<'a> { + /// Exact transfer-level replacements. + pub media_transfers: &'a [MediaTransferTransition<'a>], + /// Exact object-level replacements. + pub media_objects: &'a [MediaObjectTransition<'a>], + /// Accepted authenticated controls consumed by the transition. + pub delete_controls: &'a [DeferredControlRecord], + /// Whether presentation must be recoverable after commit. + pub presentation_changed: bool, +} + +/// One bounded group membership, announcement, authority, or deferred-control transition. +pub struct GroupStatePlan<'a> { + /// Group creations, replacements, or removals. + pub groups: &'a [GroupStateTransition<'a>], + /// Receiver-chain creations, replacements, or removals. + pub chains: &'a [GroupChainStateTransition<'a>], + /// Contact stubs authenticated by the accepted roster. + pub contacts: &'a [ContactTransition<'a>], + /// Signed authority state creations or replacements. + pub authorities: &'a [GroupAuthorityStateTransition<'a>], + /// Accepted controls consumed by this exact state transition. + pub delete_controls: &'a [DeferredControlRecord], /// Whether presentation must be recoverable after commit. pub presentation_changed: bool, } @@ -211,14 +407,20 @@ pub struct PairwiseReceivePlan<'a> { pub presentation_changed: bool, } -/// Exact prekey-vault replacement performed by an inbound handshake. +/// Exact prekey-vault replacement performed by issuance or an inbound handshake. pub struct PrekeyTransition<'a> { /// Encoded durable vault expected before the transaction. pub before: &'a [u8], - /// Encoded candidate vault after one-time-prekey consumption. + /// Encoded candidate vault after one-time-prekey issuance or consumption. pub after: &'a [u8], } +/// One newly issued prekey bundle and the vault state that owns its OPK. +pub struct PrekeyPublishPlan<'a> { + /// Exact vault replacement that makes the returned bundle usable. + pub prekeys: PrekeyTransition<'a>, +} + /// Complete durable consequences of accepting one inbound handshake. pub struct HandshakeReceivePlan<'a> { /// Optional one-time-prekey consumption. @@ -339,10 +541,22 @@ pub struct MaintenancePlan<'a> { /// The only protocol-state transaction variants exposed by the store. pub enum CommitPlan<'a> { + /// Prekey bundle issuance. + PrekeyPublish(PrekeyPublishPlan<'a>), /// Pairwise send. PairwiseSend(PairwiseSendPlan<'a>), /// Pairwise receive. PairwiseReceive(PairwiseReceivePlan<'a>), + /// Sender-key group send or late fan-out. + GroupSend(GroupSendPlan<'a>), + /// Sender-key group receive. + GroupReceive(GroupReceivePlan<'a>), + /// Local outbound attachment metadata stage. + AttachmentStage(AttachmentStagePlan<'a>), + /// Attachment lifecycle, progress, or deferred-control transition. + AttachmentState(AttachmentStatePlan<'a>), + /// Group membership, authority, chain, or deferred-control transition. + GroupState(GroupStatePlan<'a>), /// Handshake receive. HandshakeReceive(HandshakeReceivePlan<'a>), /// Receipt or authenticated pairwise control receive. @@ -426,8 +640,14 @@ impl Store { ) -> Result { self.validate_commit_plan(&plan)?; let presentation_changed = match &plan { + CommitPlan::PrekeyPublish(_) => false, CommitPlan::PairwiseSend(plan) => plan.presentation_changed, CommitPlan::PairwiseReceive(plan) => plan.presentation_changed, + CommitPlan::GroupSend(plan) => plan.presentation_changed, + CommitPlan::GroupReceive(plan) => plan.presentation_changed, + CommitPlan::AttachmentStage(plan) => plan.presentation_changed, + CommitPlan::AttachmentState(plan) => plan.presentation_changed, + CommitPlan::GroupState(plan) => plan.presentation_changed, CommitPlan::HandshakeReceive(plan) => plan.presentation_changed, CommitPlan::ReceiptReceive(plan) => plan.presentation_changed, CommitPlan::Maintenance(plan) => plan.presentation_changed, @@ -451,8 +671,14 @@ impl Store { .store .check_commit_failpoint(CommitPoint::AfterBegin)?; match plan { + CommitPlan::PrekeyPublish(plan) => writer.prekey_publish(&plan)?, CommitPlan::PairwiseSend(plan) => writer.pairwise_send(&plan)?, CommitPlan::PairwiseReceive(plan) => writer.pairwise_receive(&plan)?, + CommitPlan::GroupSend(plan) => writer.group_send(&plan)?, + CommitPlan::GroupReceive(plan) => writer.group_receive(&plan)?, + CommitPlan::AttachmentStage(plan) => writer.attachment_stage(&plan)?, + CommitPlan::AttachmentState(plan) => writer.attachment_state(&plan)?, + CommitPlan::GroupState(plan) => writer.group_state(&plan)?, CommitPlan::HandshakeReceive(plan) => writer.handshake_receive(&plan)?, CommitPlan::ReceiptReceive(plan) => writer.receipt_receive(&plan)?, CommitPlan::Maintenance(plan) => writer.maintenance(&plan)?, @@ -533,14 +759,32 @@ impl Store { fn validate_commit_plan(&self, plan: &CommitPlan<'_>) -> Result<()> { match plan { + CommitPlan::PrekeyPublish(plan) => self.validate_prekey_publish(plan), CommitPlan::PairwiseSend(plan) => self.validate_pairwise_send(plan), CommitPlan::PairwiseReceive(plan) => self.validate_pairwise_receive(plan), + CommitPlan::GroupSend(plan) => self.validate_group_send(plan), + CommitPlan::GroupReceive(plan) => self.validate_group_receive(plan), + CommitPlan::AttachmentStage(plan) => self.validate_attachment_stage(plan), + CommitPlan::AttachmentState(plan) => self.validate_attachment_state(plan), + CommitPlan::GroupState(plan) => self.validate_group_state(plan), CommitPlan::HandshakeReceive(plan) => self.validate_handshake_receive(plan), CommitPlan::ReceiptReceive(plan) => self.validate_receipt_receive(plan), CommitPlan::Maintenance(plan) => self.validate_maintenance(plan), } } + fn validate_prekey_publish(&self, plan: &PrekeyPublishPlan<'_>) -> Result<()> { + if plan.prekeys.before.is_empty() + || plan.prekeys.after.is_empty() + || plan.prekeys.before == plan.prekeys.after + || self.get_prekeys()?.as_ref().map(|value| value.as_slice()) + != Some(plan.prekeys.before) + { + return Err(StoreError::InvalidTransition); + } + Ok(()) + } + fn validate_pairwise_send(&self, plan: &PairwiseSendPlan<'_>) -> Result<()> { if plan.sessions.is_empty() || plan.sessions.len() > MAX_PAIRWISE_COMMIT_DEVICES @@ -553,11 +797,18 @@ impl Store { plan.deliveries.len(), plan.delivery_updates.len(), plan.queue.len(), + plan.groups.len(), + plan.authorities.len(), usize::from(plan.scheduled.is_some()), plan.clear_capabilities.len(), plan.clear_reset_markers.len(), usize::from(plan.ephemeral.is_some()), + plan.media_transfers.len(), + plan.media_objects.len(), + plan.delete_controls.len(), ])? > MAX_COMMIT_MUTATIONS + || plan.groups.len() > MAX_GROUP_AUTHORITY_MEMBERS + || plan.authorities.len() > MAX_GROUP_AUTHORITY_MEMBERS { return Err(StoreError::InvalidTransition); } @@ -584,6 +835,38 @@ impl Store { for transition in plan.sessions { self.validate_session_transition(transition)?; } + for transition in plan.groups { + self.validate_group_transition(transition)?; + validate_group_control_transition(transition, plan.queue)?; + } + let authority_ids = plan + .authorities + .iter() + .filter_map(|transition| transition.after.map(|authority| authority.group)) + .collect::>(); + if authority_ids.len() != plan.authorities.len() + || (!plan.authorities.is_empty() + && !plan + .queue + .iter() + .any(|item| item.envelope.kind == EnvelopeKind::GroupControl)) + { + return Err(StoreError::InvalidTransition); + } + for transition in plan.authorities { + let (Some(before), Some(after)) = (transition.before, transition.after) else { + return Err(StoreError::InvalidTransition); + }; + if before.group != after.group + || before.state_id != after.state_id + || before.state_payload != after.state_payload + || before.consumed_requests == after.consumed_requests + || self.get_group_authority(&before.group)?.as_ref() != Some(before) + || !valid_group_authority_record(after) + { + return Err(StoreError::InvalidTransition); + } + } if plan.message.is_some() && plan.message_update.is_some() { return Err(StoreError::InvalidTransition); } @@ -626,6 +909,9 @@ impl Store { { return Err(StoreError::InvalidTransition); } + if history_id.is_some() && !plan.groups.is_empty() { + return Err(StoreError::InvalidTransition); + } if plan.deliveries.iter().any(|delivery| { Some(delivery.message) != history_id || delivery.state != DeliveryState::Queued }) { @@ -728,6 +1014,46 @@ impl Store { return Err(StoreError::InvalidTransition); } } + if let Some(ephemeral) = plan.ephemeral { + if Some(ephemeral.content_id) != history_id + || ephemeral.conversation + != crate::EphemeralConversation::Pairwise(plan.message.map_or_else( + || { + plan.message_update + .as_ref() + .expect("history exists") + .after + .peer + }, + |message| message.peer, + )) + || self + .get_ephemeral_record( + &ephemeral.conversation, + &ephemeral.author, + &ephemeral.content_id, + )? + .is_some() + { + return Err(StoreError::InvalidTransition); + } + } + for transition in plan.media_transfers { + self.validate_media_transfer_transition(transition)?; + if history_id + .is_some_and(|history_id| transition.after.manifest_content_id != history_id) + { + return Err(StoreError::InvalidTransition); + } + } + for transition in plan.media_objects { + self.validate_media_object_transition(transition)?; + } + for control in plan.delete_controls { + if self.get_deferred_control(&control.content_id)?.as_ref() != Some(control) { + return Err(StoreError::InvalidTransition); + } + } let reset_markers = self.reset_markers()?.into_iter().collect::>(); if plan .clear_reset_markers @@ -772,19 +1098,685 @@ impl Store { if let Some(message) = plan.message { self.validate_new_message(message)?; } - self.validate_pending(plan.source_pending)?; - self.validate_new_media(plan.media_transfers, plan.media_objects)?; - let accepted_state = plan.message.is_some() - || plan.ephemeral.is_some() - || !plan.media_transfers.is_empty() - || !plan.media_objects.is_empty() - || plan.capabilities.is_some(); - if (plan.receipt_replay && plan.queue.is_empty()) - || (accepted_state && (!plan.receipt_replay || plan.queue.is_empty())) - || (self.is_seen(&plan.content_id)? - && (accepted_state || !plan.receipt_replay || plan.queue.is_empty())) - { - return Err(StoreError::InvalidTransition); + self.validate_pending(plan.source_pending)?; + self.validate_new_media(plan.media_transfers, plan.media_objects)?; + let accepted_state = plan.message.is_some() + || plan.ephemeral.is_some() + || !plan.media_transfers.is_empty() + || !plan.media_objects.is_empty() + || plan.capabilities.is_some(); + if (plan.receipt_replay && plan.queue.is_empty()) + || (accepted_state && (!plan.receipt_replay || plan.queue.is_empty())) + || (self.is_seen(&plan.content_id)? + && (accepted_state || !plan.receipt_replay || plan.queue.is_empty())) + { + return Err(StoreError::InvalidTransition); + } + Ok(()) + } + + fn validate_group_send(&self, plan: &GroupSendPlan<'_>) -> Result<()> { + if plan.message.is_some() == plan.message_update.is_some() + || plan.queue.len() > MAX_GROUP_COMMIT_QUEUE_ROWS + || mutation_count([ + usize::from(plan.group.is_some()), + usize::from(plan.message.is_some()), + usize::from(plan.message_update.is_some()), + plan.deliveries.len(), + plan.delivery_updates.len(), + plan.queue.len(), + usize::from(plan.scheduled.is_some()), + usize::from(plan.ephemeral.is_some()), + plan.media_transfers.len(), + plan.delete_chains.len(), + usize::from(plan.authority.is_some()), + ])? > MAX_GROUP_COMMIT_MUTATIONS + { + return Err(StoreError::InvalidTransition); + } + + let (message_id, group_id, message_after, retained_wire) = + if let Some(message) = plan.message { + self.validate_new_group_message(message)?; + if message.direction != Direction::Outbound { + return Err(StoreError::InvalidTransition); + } + (message.id, message.group, message, None) + } else { + let transition = plan + .message_update + .as_ref() + .ok_or(StoreError::InvalidTransition)?; + self.validate_group_message_transition(transition)?; + if transition.after.direction != Direction::Outbound { + return Err(StoreError::InvalidTransition); + } + ( + transition.after.id, + transition.after.group, + transition.after, + transition.before.wire_body.as_deref(), + ) + }; + + if let Some(group) = &plan.group { + self.validate_group_transition(group)?; + if group.after.id != group_id + || group.before.sender_chain == group.after.sender_chain + || (plan.queue.is_empty() && message_after.wire_body.is_none()) + { + return Err(StoreError::InvalidTransition); + } + } else if retained_wire.is_none() || plan.message.is_some() { + // A plan without a sender-chain transition may only fan out an + // already durable ciphertext retained by the exact prior row. + return Err(StoreError::InvalidTransition); + } + let stored_distribution_group = if plan.group.is_none() { + self.get_group(&group_id)? + } else { + None + }; + let distribution_group = plan + .group + .as_ref() + .map(|group| group.before) + .or(stored_distribution_group.as_ref()) + .ok_or(StoreError::InvalidTransition)?; + let eligible_accounts = distribution_group + .members + .iter() + .filter(|member| member.peer != message_after.sender) + .map(|member| member.peer) + .collect::>(); + let history_accounts = message_after + .deliveries + .iter() + .map(|delivery| delivery.peer) + .collect::>(); + if history_accounts.len() != message_after.deliveries.len() + || history_accounts != eligible_accounts + || plan.queue.len() + > eligible_accounts + .len() + .saturating_mul(MAX_PAIRWISE_COMMIT_DEVICES) + { + return Err(StoreError::InvalidTransition); + } + + let wire = plan + .queue + .first() + .map(|item| item.envelope.body.as_slice()) + .or(message_after.wire_body.as_deref()) + .or(retained_wire) + .ok_or(StoreError::InvalidTransition)?; + if plan.queue.iter().any(|item| { + item.msg_id.is_some() + || item.group_msg_id != Some(message_id) + || item.envelope.kind != EnvelopeKind::GroupMessage + || item.envelope.body.as_slice() != wire + }) || message_after + .wire_body + .as_deref() + .is_some_and(|retained| retained != wire) + || retained_wire.is_some_and(|retained| retained != wire) + { + return Err(StoreError::InvalidTransition); + } + + let queue_ids = plan + .queue + .iter() + .map(|item| item.envelope.content_id()) + .collect::>(); + if queue_ids.len() != plan.queue.len() { + return Err(StoreError::InvalidTransition); + } + let existing_queue = self.queue_all()?; + let existing_deliveries = self.message_device_deliveries(&message_id)?; + let mut account_device_counts = HashMap::new(); + for delivery in &existing_deliveries { + let count = account_device_counts + .entry(delivery.account) + .or_insert(0usize); + *count = count.saturating_add(1); + } + if account_device_counts + .values() + .any(|count| *count > MAX_PAIRWISE_COMMIT_DEVICES) + { + return Err(StoreError::InvalidTransition); + } + let mut devices = HashSet::new(); + for delivery in plan.deliveries { + let count = account_device_counts + .entry(delivery.account) + .or_insert(0usize); + *count = count.saturating_add(1); + if delivery.message != message_id + || delivery.state != DeliveryState::Queued + || !eligible_accounts.contains(&delivery.account) + || *count > MAX_PAIRWISE_COMMIT_DEVICES + || !devices.insert(delivery.device) + || existing_deliveries + .iter() + .any(|existing| existing.device == delivery.device) + { + return Err(StoreError::InvalidTransition); + } + } + for transition in plan.delivery_updates { + self.validate_delivery_transition(transition)?; + if transition.after.message != message_id + || transition.after.state != DeliveryState::Queued + || transition.before.account != transition.after.account + || !eligible_accounts.contains(&transition.after.account) + || !devices.insert(transition.after.device) + { + return Err(StoreError::InvalidTransition); + } + } + for item in plan.queue { + let wire_id = item.envelope.content_id(); + let owners = plan + .deliveries + .iter() + .filter(|delivery| { + delivery.message == message_id + && delivery.device == item.peer + && delivery.wire_id == Some(wire_id) + }) + .count() + + plan + .delivery_updates + .iter() + .filter(|transition| { + transition.after.message == message_id + && transition.after.device == item.peer + && transition.after.wire_id == Some(wire_id) + }) + .count(); + if owners != 1 { + return Err(StoreError::InvalidTransition); + } + } + for delivery in plan.deliveries.iter().chain( + plan.delivery_updates + .iter() + .map(|transition| transition.after), + ) { + if let Some(wire_id) = delivery.wire_id { + let owned = plan.queue.iter().any(|item| { + item.peer == delivery.device + && item.group_msg_id == Some(message_id) + && item.envelope.content_id() == wire_id + }) || existing_queue.iter().any(|(_, item)| { + item.peer == delivery.device + && item.group_msg_id == Some(message_id) + && item.envelope.content_id() == wire_id + }); + if !owned { + return Err(StoreError::InvalidTransition); + } + } + } + + if message_after + .deliveries + .iter() + .any(|delivery| delivery.peer == message_after.sender) + { + return Err(StoreError::InvalidTransition); + } + if let Some(scheduled) = plan.scheduled { + if self.get_scheduled_message(&scheduled.id)?.as_ref() != Some(scheduled) + || scheduled.id != message_id + { + return Err(StoreError::InvalidTransition); + } + } + if let Some(ephemeral) = plan.ephemeral { + if ephemeral.content_id != message_id + || ephemeral.conversation != crate::EphemeralConversation::Group(group_id) + || self + .get_ephemeral_record( + &ephemeral.conversation, + &ephemeral.author, + &ephemeral.content_id, + )? + .is_some() + { + return Err(StoreError::InvalidTransition); + } + } + for transition in plan.media_transfers { + self.validate_media_transfer_transition(transition)?; + if transition.after.manifest_content_id != message_id { + return Err(StoreError::InvalidTransition); + } + } + let deleted_chain_ids = plan + .delete_chains + .iter() + .map(|transition| (transition.group, transition.peer)) + .collect::>(); + if deleted_chain_ids.len() != plan.delete_chains.len() { + return Err(StoreError::InvalidTransition); + } + for transition in plan.delete_chains { + let current = self.get_group_chain(&transition.group, &transition.peer)?; + if transition.after.is_some() + || transition.before.is_none() + || current.as_ref().map(|chain| chain.as_slice()) != transition.before + || transition.group != group_id + || plan.group.as_ref().is_none_or(|group| { + group + .after + .members + .iter() + .any(|member| member.peer == transition.peer) + }) + { + return Err(StoreError::InvalidTransition); + } + } + if let Some(authority) = &plan.authority { + self.validate_group_authority_transition(authority)?; + if authority.after.group != group_id || authority.after.state_id != message_id { + return Err(StoreError::InvalidTransition); + } + } + Ok(()) + } + + fn validate_group_receive(&self, plan: &GroupReceivePlan<'_>) -> Result<()> { + self.validate_group_chain_transition(&plan.chain)?; + self.validate_session_transition(&plan.receipt_session)?; + if plan.content_id == [0u8; 16] + || self.is_seen(&plan.content_id)? + || plan.queue.is_empty() + || plan.queue.len() > MAX_COMMIT_QUEUE_ROWS + || plan.queue.iter().any(|item| { + item.peer != plan.receipt_session.peer_device + || item.msg_id.is_some() + || item.group_msg_id.is_some() + || item.envelope.kind != EnvelopeKind::Receipt + }) + || mutation_count([ + 1, + 1, + usize::from(plan.message.is_some()), + usize::from(plan.ephemeral.is_some()), + plan.media_transfers.len(), + plan.media_objects.len(), + plan.queue.len(), + 1, + 1, + usize::from(plan.source_pending.is_some()), + ])? > MAX_COMMIT_MUTATIONS + { + return Err(StoreError::InvalidTransition); + } + if let Some(message) = plan.message { + self.validate_new_group_message(message)?; + if message.direction != Direction::Inbound + || message.group != plan.chain.group + || message.sender != plan.chain.peer + || !message.deliveries.is_empty() + || message.wire_body.is_some() + { + return Err(StoreError::InvalidTransition); + } + } + if let Some(ephemeral) = plan.ephemeral { + if ephemeral.conversation != crate::EphemeralConversation::Group(plan.chain.group) + || ephemeral.author != plan.chain.peer + { + return Err(StoreError::InvalidTransition); + } + } + self.validate_new_media(plan.media_transfers, plan.media_objects)?; + self.validate_pending(plan.source_pending) + } + + fn validate_attachment_stage(&self, plan: &AttachmentStagePlan<'_>) -> Result<()> { + if plan.message.is_some() == plan.group_message.is_some() + || plan.media_transfers.is_empty() + || plan.media_objects.is_empty() + || mutation_count([ + usize::from(plan.message.is_some()), + usize::from(plan.group_message.is_some()), + plan.media_transfers.len(), + plan.media_objects.len(), + usize::from(plan.ephemeral.is_some()), + ])? > MAX_ATTACHMENT_STAGE_MUTATIONS + { + return Err(StoreError::InvalidTransition); + } + self.validate_new_media(plan.media_transfers, plan.media_objects)?; + + let (content_id, author, conversation, peers) = if let Some(message) = plan.message { + self.validate_new_message(message)?; + if message.direction != Direction::Outbound + || message.state != DeliveryState::Queued + || message.wire_id.is_some() + || message.body.is_empty() + || plan.media_transfers.len() != 1 + { + return Err(StoreError::InvalidTransition); + } + ( + message.id, + None, + crate::EphemeralConversation::Pairwise(message.peer), + HashSet::from([message.peer]), + ) + } else { + let message = plan.group_message.ok_or(StoreError::InvalidTransition)?; + self.validate_new_group_message(message)?; + let peers = message + .deliveries + .iter() + .map(|delivery| delivery.peer) + .collect::>(); + if message.direction != Direction::Outbound + || message.wire_body.is_some() + || message.body.is_empty() + || peers.len() != message.deliveries.len() + || peers.len() != plan.media_transfers.len() + || message.deliveries.iter().any(|delivery| { + delivery.peer == message.sender + || delivery.wire_id.is_some() + || delivery.state != DeliveryState::Queued + }) + { + return Err(StoreError::InvalidTransition); + } + ( + message.id, + Some(message.sender), + crate::EphemeralConversation::Group(message.group), + peers, + ) + }; + + let transfer_ids = plan + .media_transfers + .iter() + .map(|transfer| transfer.local_id) + .collect::>(); + if plan.media_transfers.iter().any(|transfer| { + transfer.direction != crate::MediaDirection::Outbound + || transfer.manifest_content_id != content_id + || transfer.state != crate::MediaTransferState::Queued + || !peers.contains(&transfer.peer) + || author.is_some_and(|sender| { + transfer.scope != crate::MediaScope::Group + || transfer.scope_id + != plan.group_message.expect("group stage selected").group + || transfer.manifest_author != sender + }) + || author.is_none() && transfer.scope != crate::MediaScope::Pairwise + }) || plan.media_objects.iter().any(|object| { + !transfer_ids.contains(&object.transfer_id) + || object.state != crate::MediaTransferState::Queued + }) { + return Err(StoreError::InvalidTransition); + } + + if let Some(ephemeral) = plan.ephemeral { + let ephemeral_transfers = ephemeral + .transfer_ids + .iter() + .copied() + .collect::>(); + if ephemeral.content_id != content_id + || ephemeral.conversation != conversation + || ephemeral.state != EphemeralState::Active + || ephemeral_transfers != transfer_ids + || self + .get_ephemeral_record( + &ephemeral.conversation, + &ephemeral.author, + &ephemeral.content_id, + )? + .is_some() + { + return Err(StoreError::InvalidTransition); + } + } + Ok(()) + } + + fn validate_attachment_state(&self, plan: &AttachmentStatePlan<'_>) -> Result<()> { + let count = mutation_count([ + plan.media_transfers.len(), + plan.media_objects.len(), + plan.delete_controls.len(), + ])?; + if count == 0 || count > MAX_MAINTENANCE_TRANSITIONS { + return Err(StoreError::MaintenanceBounds); + } + let transfer_ids = plan + .media_transfers + .iter() + .map(|transition| transition.after.local_id) + .collect::>(); + let object_ids = plan + .media_objects + .iter() + .map(|transition| transition.after.local_id) + .collect::>(); + let control_ids = plan + .delete_controls + .iter() + .map(|control| control.content_id) + .collect::>(); + if transfer_ids.len() != plan.media_transfers.len() + || object_ids.len() != plan.media_objects.len() + || control_ids.len() != plan.delete_controls.len() + { + return Err(StoreError::InvalidTransition); + } + for transition in plan.media_transfers { + self.validate_media_transfer_transition(transition)?; + } + for transition in plan.media_objects { + self.validate_media_object_transition(transition)?; + } + for control in plan.delete_controls { + if self.get_deferred_control(&control.content_id)?.as_ref() != Some(control) { + return Err(StoreError::InvalidTransition); + } + } + Ok(()) + } + + fn validate_group_state(&self, plan: &GroupStatePlan<'_>) -> Result<()> { + let count = mutation_count([ + plan.groups.len(), + plan.chains.len(), + plan.contacts.len(), + plan.authorities.len(), + plan.delete_controls.len(), + ])?; + if count == 0 || count > MAX_GROUP_STATE_MUTATIONS { + return Err(StoreError::MaintenanceBounds); + } + + let mut group_ids = HashSet::new(); + for transition in plan.groups { + let id = match (transition.before, transition.after) { + (None, None) => return Err(StoreError::InvalidTransition), + (None, Some(after)) => after.id, + (Some(before), None) => before.id, + (Some(before), Some(after)) => { + if before == after || before.id != after.id { + return Err(StoreError::InvalidTransition); + } + before.id + } + }; + if !group_ids.insert(id) + || self.get_group(&id)?.as_ref() != transition.before + || transition + .after + .is_some_and(|group| !valid_group_record(group)) + { + return Err(StoreError::InvalidTransition); + } + } + + let mut chain_ids = HashSet::new(); + for transition in plan.chains { + let current = self.get_group_chain(&transition.group, &transition.peer)?; + if !chain_ids.insert((transition.group, transition.peer)) + || (transition.before.is_none() && transition.after.is_none()) + || transition.before == transition.after + || transition.before.is_some_and(<[u8]>::is_empty) + || transition.after.is_some_and(<[u8]>::is_empty) + || current.as_ref().map(|value| value.as_slice()) != transition.before + { + return Err(StoreError::InvalidTransition); + } + } + + let mut contact_ids = HashSet::new(); + for transition in plan.contacts { + if !contact_ids.insert(transition.after.peer) + || transition.before == Some(transition.after) + || transition + .before + .is_some_and(|before| before.peer != transition.after.peer) + || transition.after.identity.is_empty() + || transition.after.identity.len() > MAX_GROUP_MEMBER_IDENTITY_LEN + || self.get_contact(&transition.after.peer)?.as_ref() != transition.before + { + return Err(StoreError::InvalidTransition); + } + } + + let mut authority_ids = HashSet::new(); + for transition in plan.authorities { + let group = match (transition.before, transition.after) { + (None, None) => return Err(StoreError::InvalidTransition), + (None, Some(after)) => after.group, + (Some(before), None) => before.group, + (Some(before), Some(after)) => { + if before == after || before.group != after.group { + return Err(StoreError::InvalidTransition); + } + before.group + } + }; + if !authority_ids.insert(group) + || self.get_group_authority(&group)?.as_ref() != transition.before + || transition + .after + .is_some_and(|authority| !valid_group_authority_record(authority)) + { + return Err(StoreError::InvalidTransition); + } + } + + let control_ids = plan + .delete_controls + .iter() + .map(|control| control.content_id) + .collect::>(); + if control_ids.len() != plan.delete_controls.len() { + return Err(StoreError::InvalidTransition); + } + for control in plan.delete_controls { + if self.get_deferred_control(&control.content_id)?.as_ref() != Some(control) { + return Err(StoreError::InvalidTransition); + } + } + + for transition in plan.groups { + let id = transition + .before + .or(transition.after) + .expect("validated group transition") + .id; + let current_chains = self.group_chains(&id)?; + if let Some(final_group) = transition.after { + for (peer, _) in current_chains { + let removed = plan.chains.iter().any(|chain| { + chain.group == id && chain.peer == peer && chain.after.is_none() + }); + if !removed && !final_group.members.iter().any(|member| member.peer == peer) { + return Err(StoreError::InvalidTransition); + } + } + } else { + let deleted_chains = plan + .chains + .iter() + .filter(|chain| chain.group == id && chain.after.is_none()) + .map(|chain| chain.peer) + .collect::>(); + if deleted_chains.len() != current_chains.len() + || current_chains + .iter() + .any(|(peer, _)| !deleted_chains.contains(peer)) + || plan + .chains + .iter() + .any(|chain| chain.group == id && chain.after.is_some()) + { + return Err(StoreError::InvalidTransition); + } + let current_authority = self.get_group_authority(&id)?; + let delete_authority = plan.authorities.iter().find(|authority| { + authority.before.is_some_and(|before| before.group == id) + && authority.after.is_none() + }); + if current_authority.is_some() != delete_authority.is_some() { + return Err(StoreError::InvalidTransition); + } + } + } + + for transition in plan.chains { + if transition.after.is_some() { + let planned_group = plan.groups.iter().find_map(|candidate| { + candidate.after.filter(|group| group.id == transition.group) + }); + let current_group; + let group = if let Some(group) = planned_group { + group + } else { + current_group = self.get_group(&transition.group)?; + current_group + .as_ref() + .ok_or(StoreError::InvalidTransition)? + }; + if !group + .members + .iter() + .any(|member| member.peer == transition.peer) + { + return Err(StoreError::InvalidTransition); + } + } + } + for transition in plan.authorities { + if let Some(authority) = transition.after { + let group_exists = plan.groups.iter().any(|candidate| { + candidate + .after + .is_some_and(|group| group.id == authority.group) + }) || (plan.groups.iter().all(|candidate| { + candidate + .before + .is_none_or(|group| group.id != authority.group) + }) && self.get_group(&authority.group)?.is_some()); + if !group_exists { + return Err(StoreError::InvalidTransition); + } + } } Ok(()) } @@ -1201,6 +2193,7 @@ impl Store { fn validate_group_transition(&self, transition: &GroupTransition<'_>) -> Result<()> { if transition.before.id != transition.after.id + || !valid_group_record(transition.after) || self.get_group(&transition.before.id)?.as_ref() != Some(transition.before) { return Err(StoreError::InvalidTransition); @@ -1208,6 +2201,39 @@ impl Store { Ok(()) } + fn validate_group_chain_transition(&self, transition: &GroupChainTransition<'_>) -> Result<()> { + let current = self.get_group_chain(&transition.group, &transition.peer)?; + if transition.before.is_empty() + || transition.after.is_empty() + || transition.before == transition.after + || current.as_ref().map(|value| value.as_slice()) != Some(transition.before) + || self.get_group(&transition.group)?.is_none_or(|group| { + !group + .members + .iter() + .any(|member| member.peer == transition.peer) + }) + { + return Err(StoreError::InvalidTransition); + } + Ok(()) + } + + fn validate_group_authority_transition( + &self, + transition: &GroupAuthorityTransition<'_>, + ) -> Result<()> { + if !valid_group_authority_record(transition.after) + || transition + .before + .is_some_and(|before| before.group != transition.after.group) + || self.get_group_authority(&transition.after.group)?.as_ref() != transition.before + { + return Err(StoreError::InvalidTransition); + } + Ok(()) + } + fn validate_group_message_transition( &self, transition: &GroupMessageTransition<'_>, @@ -1230,6 +2256,17 @@ impl Store { Ok(()) } + fn validate_new_group_message(&self, message: &GroupMessageRecord) -> Result<()> { + if self + .group_messages(&message.group)? + .into_iter() + .any(|existing| existing.id == message.id) + { + return Err(StoreError::InvalidTransition); + } + Ok(()) + } + fn validate_new_media( &self, transfers: &[MediaTransferRecord], @@ -1264,6 +2301,35 @@ impl Store { Ok(()) } + fn validate_media_transfer_transition( + &self, + transition: &MediaTransferTransition<'_>, + ) -> Result<()> { + if transition.before.local_id != transition.after.local_id + || transition.before == transition.after + || self.get_media_transfer(&transition.before.local_id)? + != Some(MediaRecord::Available(transition.before.clone())) + { + return Err(StoreError::InvalidTransition); + } + Ok(()) + } + + fn validate_media_object_transition( + &self, + transition: &MediaObjectTransition<'_>, + ) -> Result<()> { + if transition.before.local_id != transition.after.local_id + || transition.before.transfer_id != transition.after.transfer_id + || transition.before == transition.after + || self.get_media_object(&transition.before.local_id)? + != Some(MediaRecord::Available(transition.before.clone())) + { + return Err(StoreError::InvalidTransition); + } + Ok(()) + } + /// Read one accepted deferred control by its envelope id. pub fn get_deferred_control( &self, @@ -1373,6 +2439,12 @@ impl CommitWriter<'_, R> { }) } + fn group_chain(&mut self, transition: &GroupChainTransition<'_>) -> Result<()> { + self.write(|store, rng| { + store.put_group_chain(&transition.group, &transition.peer, transition.after, rng) + }) + } + fn queue(&mut self, items: &[QueueItem]) -> Result<()> { for item in items { let sequence = self.write(|store, rng| store.queue_push(item, rng))?; @@ -1394,6 +2466,10 @@ impl CommitWriter<'_, R> { Ok(()) } + fn prekey_publish(&mut self, plan: &PrekeyPublishPlan<'_>) -> Result<()> { + self.write(|store, rng| store.put_prekeys(plan.prekeys.after, rng)) + } + fn pairwise_send(&mut self, plan: &PairwiseSendPlan<'_>) -> Result<()> { for transition in plan.sessions { self.session(transition)?; @@ -1419,6 +2495,17 @@ impl CommitWriter<'_, R> { self.write(|store, rng| store.put_message_device_delivery(transition.after, rng))?; } self.queue(plan.queue)?; + for transition in plan.groups { + self.write(|store, rng| store.put_group(transition.after, rng))?; + } + for transition in plan.authorities { + self.write(|store, rng| { + store.put_group_authority( + transition.after.expect("validated authority replacement"), + rng, + ) + })?; + } if let Some(scheduled) = plan.scheduled { self.write(|store, _| { if store.delete_scheduled_message(&scheduled.id)? { @@ -1437,6 +2524,23 @@ impl CommitWriter<'_, R> { if let Some(ephemeral) = plan.ephemeral { self.write(|store, rng| store.put_ephemeral_record(ephemeral, rng))?; } + for transition in plan.media_transfers { + self.write(|store, rng| store.put_media_transfer(transition.after, rng))?; + } + for transition in plan.media_objects { + self.write(|store, rng| store.put_media_object(transition.after, rng))?; + } + for control in plan.delete_controls { + self.write(|store, _| { + if store.delete_equality::( + &store_v2::ContentKey::new(control.content_id), + )? { + Ok(()) + } else { + Err(StoreError::InvalidTransition) + } + })?; + } Ok(()) } @@ -1477,6 +2581,203 @@ impl CommitWriter<'_, R> { self.pending(plan.source_pending) } + fn group_send(&mut self, plan: &GroupSendPlan<'_>) -> Result<()> { + if let Some(group) = &plan.group { + self.write(|store, rng| store.put_group(group.after, rng))?; + } + if let Some(message) = plan.message { + self.write(|store, rng| store.put_group_message(message, rng))?; + } + if let Some(transition) = &plan.message_update { + self.write(|store, rng| { + if store.update_group_message(transition.after, rng)? { + Ok(()) + } else { + Err(StoreError::InvalidTransition) + } + })?; + } + for delivery in plan.deliveries { + self.write(|store, rng| store.put_message_device_delivery(delivery, rng))?; + } + for transition in plan.delivery_updates { + self.write(|store, rng| store.put_message_device_delivery(transition.after, rng))?; + } + self.queue(plan.queue)?; + if let Some(scheduled) = plan.scheduled { + self.write(|store, _| { + if store.delete_scheduled_message(&scheduled.id)? { + Ok(()) + } else { + Err(StoreError::InvalidTransition) + } + })?; + } + if let Some(ephemeral) = plan.ephemeral { + self.write(|store, rng| store.put_ephemeral_record(ephemeral, rng))?; + } + for transition in plan.media_transfers { + self.write(|store, rng| store.put_media_transfer(transition.after, rng))?; + } + for transition in plan.delete_chains { + self.write(|store, _| { + if store.delete_equality::( + &store_v2::GroupMemberKey::new(transition.group, transition.peer), + )? { + Ok(()) + } else { + Err(StoreError::InvalidTransition) + } + })?; + } + if let Some(authority) = &plan.authority { + self.write(|store, rng| store.put_group_authority(authority.after, rng))?; + } + Ok(()) + } + + fn group_receive(&mut self, plan: &GroupReceivePlan<'_>) -> Result<()> { + self.group_chain(&plan.chain)?; + self.session(&plan.receipt_session)?; + if let Some(message) = plan.message { + self.write(|store, rng| store.put_group_message(message, rng))?; + } + if let Some(ephemeral) = plan.ephemeral { + self.write(|store, rng| store.put_ephemeral_record(ephemeral, rng))?; + } + for transfer in plan.media_transfers { + self.write(|store, rng| store.put_media_transfer(transfer, rng))?; + } + for object in plan.media_objects { + self.write(|store, rng| store.put_media_object(object, rng))?; + } + self.queue(plan.queue)?; + self.write(|store, rng| { + store.put_equality::( + &store_v2::ContentKey::new(plan.content_id), + &plan.content_id, + store_v2::IndexKeys::none(), + rng, + ) + })?; + let peer = plan.receipt_session.peer_device; + self.write(|store, rng| { + store.put_receipt_replay(&plan.content_id, &peer, plan.received_at, rng) + })?; + self.pending(plan.source_pending) + } + + fn attachment_stage(&mut self, plan: &AttachmentStagePlan<'_>) -> Result<()> { + if let Some(message) = plan.message { + self.write(|store, rng| store.put_message(message, rng))?; + self.records.messages.push(message.id); + } + if let Some(message) = plan.group_message { + self.write(|store, rng| store.put_group_message(message, rng))?; + } + for transfer in plan.media_transfers { + self.write(|store, rng| store.put_media_transfer(transfer, rng))?; + } + for object in plan.media_objects { + self.write(|store, rng| store.put_media_object(object, rng))?; + } + if let Some(ephemeral) = plan.ephemeral { + self.write(|store, rng| store.put_ephemeral_record(ephemeral, rng))?; + } + Ok(()) + } + + fn attachment_state(&mut self, plan: &AttachmentStatePlan<'_>) -> Result<()> { + for transition in plan.media_transfers { + self.write(|store, rng| store.put_media_transfer(transition.after, rng))?; + } + for transition in plan.media_objects { + self.write(|store, rng| store.put_media_object(transition.after, rng))?; + } + for control in plan.delete_controls { + self.write(|store, _| { + if store.delete_equality::( + &store_v2::ContentKey::new(control.content_id), + )? { + Ok(()) + } else { + Err(StoreError::InvalidTransition) + } + })?; + } + Ok(()) + } + + fn group_state(&mut self, plan: &GroupStatePlan<'_>) -> Result<()> { + for transition in plan.contacts { + self.write(|store, rng| store.put_contact(transition.after, rng))?; + } + for transition in plan.groups { + if let Some(group) = transition.after { + self.write(|store, rng| store.put_group(group, rng))?; + } else { + let group = transition.before.expect("validated group deletion").id; + self.write(|store, _| { + if store + .delete_equality::(&store_v2::GroupKey::new(group))? + { + Ok(()) + } else { + Err(StoreError::InvalidTransition) + } + })?; + } + } + for transition in plan.chains { + if let Some(chain) = transition.after { + self.write(|store, rng| { + store.put_group_chain(&transition.group, &transition.peer, chain, rng) + })?; + } else { + self.write(|store, _| { + if store.delete_equality::( + &store_v2::GroupMemberKey::new(transition.group, transition.peer), + )? { + Ok(()) + } else { + Err(StoreError::InvalidTransition) + } + })?; + } + } + for transition in plan.authorities { + if let Some(authority) = transition.after { + self.write(|store, rng| store.put_group_authority(authority, rng))?; + } else { + let group = transition + .before + .expect("validated authority deletion") + .group; + self.write(|store, _| { + if store.delete_equality::( + &store_v2::GroupKey::new(group), + )? { + Ok(()) + } else { + Err(StoreError::InvalidTransition) + } + })?; + } + } + for control in plan.delete_controls { + self.write(|store, _| { + if store.delete_equality::( + &store_v2::ContentKey::new(control.content_id), + )? { + Ok(()) + } else { + Err(StoreError::InvalidTransition) + } + })?; + } + Ok(()) + } + fn handshake_receive(&mut self, plan: &HandshakeReceivePlan<'_>) -> Result<()> { if let Some(prekeys) = &plan.prekeys { self.write(|store, rng| store.put_prekeys(prekeys.after, rng))?; @@ -1821,6 +3122,97 @@ fn session_eq(left: Option<&Session>, right: Option<&Session>) -> Result { } } +fn valid_group_record(group: &GroupRecord) -> bool { + let member_ids = group + .members + .iter() + .map(|member| member.peer) + .collect::>(); + let pending_ids = group + .pending + .iter() + .map(|pending| pending.peer) + .collect::>(); + group.id != [0u8; 32] + && !group.name.is_empty() + && group.name.len() <= MAX_GROUP_NAME_LEN + && !group.members.is_empty() + && group.members.len() <= MAX_GROUP_AUTHORITY_MEMBERS + && member_ids.len() == group.members.len() + && group.members.iter().all(|member| { + !member.identity.is_empty() && member.identity.len() <= MAX_GROUP_MEMBER_IDENTITY_LEN + }) + && group.secret != [0u8; 32] + && !group.sender_chain.is_empty() + && group.pending.len() <= MAX_GROUP_AUTHORITY_MEMBERS + && pending_ids.len() == group.pending.len() + && group + .pending + .iter() + .all(|pending| member_ids.contains(&pending.peer)) +} + +fn valid_group_authority_record(authority: &GroupAuthorityRecord) -> bool { + let consumed = authority + .consumed_requests + .iter() + .copied() + .collect::>(); + authority.group != [0u8; 32] + && authority.state_id != [0u8; 16] + && !authority.state_payload.is_empty() + && authority.consumed_requests.len() <= MAX_GROUP_ADMIN_REQUESTS + && consumed.len() == authority.consumed_requests.len() +} + +fn validate_group_control_transition( + transition: &GroupTransition<'_>, + queue: &[QueueItem], +) -> Result<()> { + let before = transition.before; + let after = transition.after; + if before.name != after.name + || before.creator != after.creator + || before.members != after.members + || before.secret != after.secret + || before.prev_secret != after.prev_secret + || before.generation != after.generation + || before.sender_chain != after.sender_chain + || before.sent_since_rotation != after.sent_since_rotation + || before.pending.len() != after.pending.len() + { + return Err(StoreError::InvalidTransition); + } + let mut changed = 0usize; + for (prior, candidate) in before.pending.iter().zip(&after.pending) { + if prior.peer != candidate.peer + || prior.key_id != candidate.key_id + || prior.chain_key != candidate.chain_key + || prior.iteration != candidate.iteration + { + return Err(StoreError::InvalidTransition); + } + if prior.wire_id == candidate.wire_id && prior.last_sent == candidate.last_sent { + continue; + } + changed += 1; + if candidate.last_sent == 0 + || candidate.wire_id.is_none_or(|wire_id| { + !queue.iter().any(|item| { + item.envelope.kind == EnvelopeKind::GroupControl + && item.envelope.content_id() == wire_id + }) + }) + { + return Err(StoreError::InvalidTransition); + } + } + if changed != 1 { + return Err(StoreError::InvalidTransition); + } + Ok(()) +} + fn mutation_count(counts: [usize; N]) -> Result { counts.into_iter().try_fold(0usize, |sum, count| { sum.checked_add(count).ok_or(StoreError::RecordBounds) diff --git a/crates/kult-store/src/lib.rs b/crates/kult-store/src/lib.rs index 831c2a9..35ebfe6 100644 --- a/crates/kult-store/src/lib.rs +++ b/crates/kult-store/src/lib.rs @@ -44,17 +44,22 @@ mod scheduled; mod store_v2; pub use backup::BACKUP_MAGIC; -#[cfg(feature = "test-failpoints")] -pub use commit::{CommitFailpoint, CommitFailure}; pub use commit::{ - CommitPlan, CommitReceipt, CommittedRecordIds, ContactDeviceDelete, DeferredControlKind, - DeferredControlRecord, DeliveryTransition, EphemeralTransition, GroupMessageDelete, - GroupMessageTransition, GroupTransition, HandshakeReceivePlan, MaintenancePlan, MediaDelete, - MessageDelete, MessageTransition, PairwiseReceivePlan, PairwiseSendPlan, PendingDelete, - PrekeyTransition, QueueDelete, QueueTransition, ReceiptReceivePlan, SessionTransition, - MAX_COMMIT_MUTATIONS, MAX_COMMIT_QUEUE_ROWS, MAX_DEFERRED_CONTROLS, - MAX_MAINTENANCE_TRANSITIONS, MAX_PAIRWISE_COMMIT_DEVICES, + AttachmentStagePlan, AttachmentStatePlan, CommitPlan, CommitReceipt, CommittedRecordIds, + ContactDeviceDelete, ContactTransition, DeferredControlKind, DeferredControlRecord, + DeliveryTransition, EphemeralTransition, GroupAuthorityStateTransition, + GroupAuthorityTransition, GroupChainStateTransition, GroupChainTransition, GroupMessageDelete, + GroupMessageTransition, GroupReceivePlan, GroupSendPlan, GroupStatePlan, GroupStateTransition, + GroupTransition, HandshakeReceivePlan, MaintenancePlan, MediaDelete, MediaObjectTransition, + MediaTransferTransition, MessageDelete, MessageTransition, PairwiseReceivePlan, + PairwiseSendPlan, PendingDelete, PrekeyPublishPlan, PrekeyTransition, QueueDelete, + QueueTransition, ReceiptReceivePlan, SessionTransition, MAX_ATTACHMENT_STAGE_MUTATIONS, + MAX_COMMIT_MUTATIONS, MAX_COMMIT_QUEUE_ROWS, MAX_DEFERRED_CONTROLS, MAX_GROUP_COMMIT_MUTATIONS, + MAX_GROUP_COMMIT_QUEUE_ROWS, MAX_GROUP_STATE_MUTATIONS, MAX_MAINTENANCE_TRANSITIONS, + MAX_PAIRWISE_COMMIT_DEVICES, }; +#[cfg(feature = "test-failpoints")] +pub use commit::{CommitFailpoint, CommitFailure}; pub use devices::{ ContactDeviceRecord, DeviceChannelRecord, DeviceStateRecord, DeviceTransferGroup, DeviceTransferSelection, DeviceTransferSnapshot, MessageDeviceDeliveryRecord, @@ -80,9 +85,9 @@ pub use maintenance::{ MAX_MAINTENANCE_WAL_BYTES, }; pub use media::{ - MediaDirection, MediaLimits, MediaObjectRecord, MediaReconciliation, MediaRecord, MediaScope, - MediaTransferRecord, MediaTransferState, MediaUsage, DEFAULT_MEDIA_STORE_QUOTA, - MAX_MEDIA_STORE_QUOTA, + MediaDirection, MediaLimits, MediaObjectRecord, MediaReconciliation, MediaReconciliationBatch, + MediaRecord, MediaScope, MediaTransferRecord, MediaTransferState, MediaUsage, StagedMediaChunk, + DEFAULT_MEDIA_STORE_QUOTA, MAX_MEDIA_STORE_QUOTA, }; pub use note::{NoteMessageRecord, MAX_NOTE_TEXT_BYTES, NOTE_TO_SELF_CONVERSATION_ID}; pub use scheduled::{ScheduledConversation, ScheduledMessageRecord}; diff --git a/crates/kult-store/src/media.rs b/crates/kult-store/src/media.rs index 5664d84..7d267bd 100644 --- a/crates/kult-store/src/media.rs +++ b/crates/kult-store/src/media.rs @@ -19,7 +19,10 @@ use kult_protocol::{ MAX_PRIMARY_OBJECT_LEN, }; -use crate::{store_v2, Result, Store, StoreError}; +use crate::{ + store_v2, AttachmentStatePlan, CommitPlan, MediaObjectTransition, Result, Store, StoreError, + MAX_MAINTENANCE_TRANSITIONS, +}; const MEDIA_RECORD_VERSION: u8 = 1; const MEDIA_CHUNK_AD: &[u8] = b"KAT-store-chunk-v1"; @@ -230,6 +233,30 @@ pub struct MediaReconciliation { pub unknown_records: usize, } +/// One bounded candidate page of startup media-state repairs. +pub struct MediaReconciliationBatch { + /// Exact missing-file object replacements. + pub transitions: Vec<(MediaObjectRecord, MediaObjectRecord)>, + /// Unknown record versions that require conservative orphan retention. + pub unknown_records: usize, + /// Whether no additional missing-file transition remains after this page. + pub complete: bool, +} + +/// File-first candidate for one attachment chunk metadata transition. +/// +/// The sealed file is durable before this value is returned. Callers commit +/// `before` → `after` through a typed protocol plan; an uncommitted file is an +/// unreferenced orphan removed by bounded reconciliation. +pub struct StagedMediaChunk { + /// Exact object row read before staging the file. + pub before: MediaObjectRecord, + /// Candidate object row that references the staged file. + pub after: MediaObjectRecord, + /// Ciphertext-derived local address of the staged sealed file. + pub address: [u8; 32], +} + pub(crate) fn prepare_media_directory(store_path: &Path) -> Result { let name = store_path.file_name().ok_or(StoreError::NotAStore)?; let mut media_name = OsString::from(name); @@ -414,21 +441,22 @@ impl Store { Ok(()) } - /// Commit one already end-to-end-encrypted chunk as a locally sealed file. + /// Stage one already end-to-end-encrypted chunk as a locally sealed file. /// - /// The temp file is fsynced, atomically renamed, and only then is the - /// sealed address map and progress bit committed in SQLite. - pub fn commit_media_chunk( + /// The temp file is fsynced and atomically renamed before an object-row + /// candidate is returned. This method never advances SQLite metadata. + pub fn stage_media_chunk( &mut self, local_id: &[u8; 16], index: u32, sealed_chunk: &[u8], rng: &mut impl CryptoRngCore, - ) -> Result<[u8; 32]> { + ) -> Result { if sealed_chunk.len() != ATTACHMENT_SEALED_CHUNK_LEN { return Err(StoreError::MediaState); } - let mut object = self.require_object(local_id)?; + let before = self.require_object(local_id)?; + let mut object = before.clone(); let slot = usize::try_from(index).map_err(|_| StoreError::MediaState)?; if slot >= object.chunk_addresses.len() { return Err(StoreError::MediaState); @@ -436,7 +464,11 @@ impl Store { let address = bulk_hash(sealed_chunk); if let Some(existing) = object.chunk_addresses[slot] { if existing == address && self.chunk_path(&address).is_file() { - return Ok(address); + return Ok(StagedMediaChunk { + before: before.clone(), + after: before, + address, + }); } return Err(StoreError::MediaState); } @@ -495,8 +527,29 @@ impl Store { if !matches!(object.state, MediaTransferState::Complete) { object.state = MediaTransferState::Transferring; } - self.persist_object(&object, rng)?; - Ok(address) + Ok(StagedMediaChunk { + before, + after: object, + address, + }) + } + + /// Commit one staged chunk through the legacy single-row media boundary. + /// + /// Protocol paths use [`Store::stage_media_chunk`] and include the + /// returned object transition in their typed commit plan. + pub fn commit_media_chunk( + &mut self, + local_id: &[u8; 16], + index: u32, + sealed_chunk: &[u8], + rng: &mut impl CryptoRngCore, + ) -> Result<[u8; 32]> { + let staged = self.stage_media_chunk(local_id, index, sealed_chunk, rng)?; + if staged.before != staged.after { + self.persist_object(&staged.after, rng)?; + } + Ok(staged.address) } /// Read and locally unseal one end-to-end encrypted chunk record. @@ -559,6 +612,30 @@ impl Store { self.persist_object(&object, rng) } + /// Prepare an exact object transition to `Complete` after checking every + /// committed address, byte count, and final content hash. + pub fn prepare_media_complete( + &self, + local_id: &[u8; 16], + verified_hash: &[u8; 32], + ) -> Result<(MediaObjectRecord, MediaObjectRecord)> { + let before = self.require_object(local_id)?; + if &before.content_hash != verified_hash + || before.chunk_addresses.iter().any(Option::is_none) + || before.verified_bytes != before.total_len + || before + .chunk_addresses + .iter() + .flatten() + .any(|address| !self.chunk_path(address).is_file()) + { + return Err(StoreError::MediaState); + } + let mut after = before.clone(); + after.state = MediaTransferState::Complete; + Ok((before, after)) + } + /// Delete one object row and garbage-collect chunk files no longer referenced. pub fn delete_media_object(&mut self, local_id: &[u8; 16]) -> Result<()> { self.delete_equality::(&store_v2::LocalIdKey::new(*local_id))?; @@ -603,31 +680,85 @@ impl Store { /// /// Call this once after `Store::open`, when the node has supplied an RNG. pub fn reconcile_media(&mut self, rng: &mut impl CryptoRngCore) -> Result { - let mut report = MediaReconciliation { - stale_temps_removed: remove_stale_temps(&self.media_dir)?, - ..MediaReconciliation::default() + let mut missing_objects = 0usize; + let unknown_records = loop { + let batch = self.prepare_media_reconciliation(MAX_MAINTENANCE_TRANSITIONS)?; + missing_objects = missing_objects + .checked_add(batch.transitions.len()) + .ok_or(StoreError::MediaQuota)?; + if !batch.transitions.is_empty() { + let transitions = batch + .transitions + .iter() + .map(|(before, after)| MediaObjectTransition { before, after }) + .collect::>(); + self.commit_plan( + CommitPlan::AttachmentState(AttachmentStatePlan { + media_transfers: &[], + media_objects: &transitions, + delete_controls: &[], + presentation_changed: false, + }), + rng, + )?; + } + if batch.complete { + break batch.unknown_records; + } }; - let records = self.media_objects()?; - for record in records { + let mut report = self.finish_media_reconciliation(unknown_records)?; + report.missing_objects = missing_objects; + Ok(report) + } + + /// Prepare at most `limit` exact missing-file object transitions. + pub fn prepare_media_reconciliation(&self, limit: usize) -> Result { + if limit == 0 || limit > MAX_MAINTENANCE_TRANSITIONS { + return Err(StoreError::MaintenanceBounds); + } + let mut transitions = Vec::new(); + let mut unknown_records = 0usize; + let mut complete = true; + for record in self.media_objects()? { match record { - MediaRecord::Unavailable { .. } => report.unknown_records += 1, - MediaRecord::Available(mut object) => { - let missing = object + MediaRecord::Unavailable { .. } => unknown_records += 1, + MediaRecord::Available(before) => { + let missing = before .chunk_addresses .iter() .flatten() .any(|address| !self.chunk_path(address).is_file()); - if missing && object.state != MediaTransferState::Unavailable { - object.state = MediaTransferState::Unavailable; - self.persist_object(&object, rng)?; - report.missing_objects += 1; + if !missing || before.state == MediaTransferState::Unavailable { + continue; + } + if transitions.len() == limit { + complete = false; + continue; } + let mut after = before.clone(); + after.state = MediaTransferState::Unavailable; + transitions.push((before, after)); } } } - report.orphan_files_removed = - self.garbage_collect_media_files(report.unknown_records > 0)?; - Ok(report) + Ok(MediaReconciliationBatch { + transitions, + unknown_records, + complete, + }) + } + + /// Remove uncommitted filesystem residue after all metadata repairs commit. + pub fn finish_media_reconciliation( + &self, + unknown_records: usize, + ) -> Result { + Ok(MediaReconciliation { + stale_temps_removed: remove_stale_temps(&self.media_dir)?, + orphan_files_removed: self.garbage_collect_media_files(unknown_records > 0)?, + missing_objects: 0, + unknown_records, + }) } /// Remove sealed chunk files no longer referenced by committed metadata. diff --git a/docs/00-start-here.md b/docs/00-start-here.md index 59db6b2..3c0a36a 100644 --- a/docs/00-start-here.md +++ b/docs/00-start-here.md @@ -170,6 +170,7 @@ Platform build instructions: | when live audio calls work—and when they deliberately do not | [Live Audio Calls](23-live-audio-calls.md) | | how a release is validated locally before any hosted run | [Local Release Gate](24-local-release-gate.md) | | what evidence is required before stable or wire v1 | [Stabilization Program](29-stabilization-program.md) | +| which persisted protocol transitions are atomic and which remain open | [Atomic Transition Inventory](34-atomic-transition-inventory.md) | | why a technical decision was made | [ADR Index](adr/README.md) | ## How can I help? diff --git a/docs/03-architecture.md b/docs/03-architecture.md index ef10170..123ca93 100644 --- a/docs/03-architecture.md +++ b/docs/03-architecture.md @@ -150,10 +150,11 @@ clock advance activates it on the next tick; time-zone changes are display-only. envelope, and any consumed schedule/reset/capability state through one typed `BEGIN IMMEDIATE` transition. The UI changes only after the commit receipt; a durable presentation marker requests snapshot resynchronization after a - commit-before-notification restart. This evidence is pairwise-specific: - group sender/receiver plans and deferred group/media control follow-up remain - open under [ADR-0028](adr/0028-atomic-protocol-commits.md), so Komms does not - claim universal crash atomicity. + commit-before-notification restart. Group sender/receiver, bounded fan-out, + attachment and deferred group/media follow-up use the corresponding typed + transitions. The [complete inventory](34-atomic-transition-inventory.md) + records the implemented and excluded boundaries; open device-authority, + first-contact, bootstrap and relay-custody paths keep ADR-0028 Proposed. 3. **kult-protocol** serializes content, pads it to the next size bucket, and hands it to the conversation's ratchet. 4. **kult-crypto** advances the sending chain, encrypts with XChaCha20-Poly1305, and diff --git a/docs/29-stabilization-program.md b/docs/29-stabilization-program.md index 4fbb715..de9db34 100644 --- a/docs/29-stabilization-program.md +++ b/docs/29-stabilization-program.md @@ -234,7 +234,7 @@ This table prevents a prior concern from disappearing into roadmap prose. | Video, large groups, new carriers, federation, and governance expansion could distract from everyday reliability | P0-03, P2 | | Direct transport currently acknowledges volatile RAM before bounded durable admission can accept or refuse an unknown token | P0-05 | | Stable identity-derived DHT locators and public route hints permit polling and network-location correlation | P0-04, P0-05, P0-06 | -| Pairwise send/receive, handshake, receipt, and bounded maintenance now use typed atomic plans with crash injection; group sender/receiver and deferred group/media follow-up still have multi-commit crash windows | P0-03, P0-06 | +| Typed atomic plans and restart injection now cover pairwise, group, attachment, scheduled activation and bounded maintenance paths; linked-device authority/sync, first-contact admission, initial profile creation and leased relay custody remain open in the recorded inventory | P0-03, P0-06 | | Mailbox collection deletes relay custody before the endpoint durably stages and acknowledges a leased page | P0-04, P0-08 | | Unix store writer exclusion now combines a no-follow sidecar with a database-inode lock; equivalent alias resistance and hostile-filesystem qualification remain open on other supported platforms | P0-06, P0-09 | | The Unix RPC sidecar is no-follow and owner-only, but portable stale-socket replacement still requires a daemon-owned parent directory to exclude hostile rename/unlink races | P0-07, P0-09 | diff --git a/docs/31-release-evidence-ledger.md b/docs/31-release-evidence-ledger.md index eb64337..f9a29d6 100644 --- a/docs/31-release-evidence-ledger.md +++ b/docs/31-release-evidence-ledger.md @@ -1,10 +1,11 @@ # Release evidence ledger -**Ledger date:** 2026-07-26 +**Ledger date:** 2026-07-27 **Release scope:** stable-v1 candidate -**Current evidence baseline:** [`main@4fda544`](https://github.com/AndriGitDev/Komms/tree/4fda544739c0665b6a324256d858c16c1d73d992) +**Current evidence baseline:** [`main@63ef901`](https://github.com/AndriGitDev/Komms/tree/63ef901e800a5b379a7088202523437fc8f1a7d1); +the atomic pairwise change was merged by PR #81 **Baseline automated run:** [`a02b064`, CI run 197](https://github.com/AndriGitDev/Komms/actions/runs/30199264838); this successful PR-head tree is the tree merged by PR #77 @@ -35,7 +36,7 @@ accepts it and the assignment is recorded in |---|---|---|---|---|---| | **P0-01 Honest claims and evidence ledger** | Andri (FND; interim SEC/PROD) | Designed: vocabulary, frozen claim register, and this ledger exist; the repository documentation check passed in release-control CI. | **Open.** External website and repository-description corrections remain; no stable release evidence bundle exists. | [Stabilization vocabulary](29-stabilization-program.md#2-evidence-vocabulary); [stable-v1 profile](30-stable-v1-product-profile.md); [CI run 199](https://github.com/AndriGitDev/Komms/actions/runs/30202463092); this ledger; [public-copy follow-up](#4-public-copy-audit) | 2026-08-09 | | **P0-02 Name-risk assessment and recorded decision** | Andri (FND and project risk owner); qualified trademark counsel: **Unassigned** | Designed: dated founder decision, observed overlap, migration cost, cadence, and escalation triggers recorded. | **Open.** This is not legal clearance. No qualified similarity/class/jurisdiction opinion or trademark/asset policy exists. | [Name-risk decision](32-name-risk-decision.md); [brand system](28-brand-system.md) | 2026-10-26, or trigger event | -| **P0-03 Stabilized core product profile** | Andri (FND; interim PROD/SEC) | Designed: product boundary, bounds, supported-system rule, services, and exclusions frozen. Pairwise send/receive, handshake, receipt, bounded maintenance, and presentation recovery have deterministic crash evidence. | **Open.** ADR-0026 and ADR-0028 through ADR-0032 remain proposed; ADR-0028 still lacks group plans and atomic deferred group/media follow-up; the profile has not passed field or independent review. | [Stable-v1 profile](30-stable-v1-product-profile.md); [ADR-0028](adr/0028-atomic-protocol-commits.md); [commit plans](../crates/kult-store/src/commit.rs); [crash matrix](../crates/kult-node/src/atomic_tests.rs); [P0 ADR index](adr/README.md) | 2026-08-09 | +| **P0-03 Stabilized core product profile** | Andri (FND; interim PROD/SEC) | Designed: product boundary, bounds, supported-system rule, services, and exclusions frozen. Typed pairwise, group, attachment, scheduled-activation and bounded-maintenance plans have deterministic local crash/restart tests. | **Open.** ADR-0026 and ADR-0028 through ADR-0032 remain proposed. Linked-device authority/sync, first-contact admission, initial profile creation and leased relay custody remain outside the complete plan matrix; no retained branch run, field qualification or independent review has closed those gaps. | [Stable-v1 profile](30-stable-v1-product-profile.md); [atomic inventory](34-atomic-transition-inventory.md); [ADR-0028](adr/0028-atomic-protocol-commits.md); [commit plans](../crates/kult-store/src/commit.rs); [crash matrix](../crates/kult-node/src/atomic_tests.rs); [P0 ADR index](adr/README.md) | 2026-08-09 | | **P0-04 Clean-install and real-network golden path** | Andri (interim NET/PROD); independent field evaluator: **Unassigned** | Implemented with local/CI evidence for internet components and shells. | **Open.** No qualified default bootstrap/mailbox, clean-device distinct-NAT matrix, default-blackhole journey, replacement operator, or pure-core journey. | [Internet tests](../crates/kult-node/tests/internet_e2e.rs); [Alpha guide](27-alpha-testing.md); [ADR-0034](adr/0034-operator-minimized-reference-discovery.md) | 2026-08-09 | | **P0-05 Unsolicited-contact abuse admission** | Andri (interim SEC/NET/PROD); independent adversarial/usability evaluator: **Unassigned** | Designed, with automated evidence for interim byte/count bounds. | **Open.** No pre-KEM admission, durable request inbox, consent promotion, identity block implementation, or adversarial/field evidence. Current first contact can create normal contact state. | [ADR-0030](adr/0030-first-contact-admission.md); [transport bounds](05-transports.md); [protocol tests](../crates/kult-protocol/tests/protocol.rs) | 2026-08-09 | | **P0-06 Independent crypto and protocol assurance** | Andri (interim SEC); independent cryptography reviewer: **Unassigned**; independent interoperability implementer: **Unassigned** | Automated evidence for local KATs, properties, sessions, and protocol decoding. | **Open.** No external vectors, separate implementation, review scope, findings report, disposition, or residual-risk statement. P0 protocol/security ADRs remain proposed. | [Baseline CI](https://github.com/AndriGitDev/Komms/actions/runs/30199264838); [Crypto KATs](../crates/kult-crypto/tests/kat.rs); [properties](../crates/kult-crypto/tests/properties.rs); [session tests](../crates/kult-crypto/tests/session.rs); [cryptography spec](04-cryptography.md) | 2026-08-09 | @@ -57,10 +58,10 @@ the listed gap. A new stable claim requires a new identifier here. | **SV1-C02 Local identity** | Komms identity can be created without a required phone number, email address, real name, or project account. | Andri (SEC/PROD); external reviewer: **Unassigned** | Automated evidence | [Identity design](06-identity-trust.md); [node e2e](../crates/kult-node/tests/node_e2e.rs); [FFI e2e](../crates/kult-ffi/tests/ffi_e2e.rs) | Independent protocol review and supported-platform clean-install evidence | 2026-08-09 | | **SV1-C03 Pairwise confidentiality and authenticity** | Accepted contacts exchange authenticated end-to-end encrypted pairwise text; intermediaries receive sealed envelopes rather than message plaintext. | Andri (SEC); external cryptography reviewer: **Unassigned** | Test evidence | [`main@4fda544`](https://github.com/AndriGitDev/Komms/tree/4fda544739c0665b6a324256d858c16c1d73d992); [KATs](../crates/kult-crypto/tests/kat.rs); [node e2e](../crates/kult-node/tests/node_e2e.rs); [pairwise crash matrix](../crates/kult-node/src/atomic_tests.rs) | P0-06 external vectors, review, and interoperability evidence | 2026-08-09 | | **SV1-C04 Contact establishment** | Intentional Connect codes and bounded message requests establish contact without making a project service the identity authority. | Andri (SEC/NET/PROD); external evaluators: **Unassigned** | Designed | [ADR-0030](adr/0030-first-contact-admission.md); [ADR-0031](adr/0031-capability-scoped-dht-discovery.md) | Implement, accept, adversarially test, and field-qualify v2 contact flow | 2026-08-09 | -| **SV1-C05 Pairwise text bounds and atomicity** | One stable-v1 text event carries at most 65,507 UTF-8 bytes and changes visible/delivery state only through its complete atomic transition. | Andri (SEC/PROD); external reviewer: **Unassigned** | Test evidence | [Content codec](../crates/kult-protocol/src/content.rs); [ADR-0028](adr/0028-atomic-protocol-commits.md); [typed commit plans](../crates/kult-store/src/commit.rs); [pairwise crash matrix](../crates/kult-node/src/atomic_tests.rs); [node e2e](../crates/kult-node/tests/node_e2e.rs) | Independent protocol review and supported-platform sudden-power-loss qualification | 2026-08-09 | -| **SV1-C06 Bounded groups** | Stable-v1 groups support at most 64 active accounts and authenticate the claimed sender separately for every recipient. | Andri (SEC/PROD); external reviewer: **Unassigned** | Automated evidence for current bounded groups; required origin property designed | [Group tests](../crates/kult-node/tests/groups_e2e.rs); [ADR-0029](adr/0029-recipient-authenticated-groups.md) | Implement recipient origin, atomic group transitions, upgrade path, malicious-member evidence, and independent review | 2026-08-09 | -| **SV1-C07 Bounded attachments** | Consented attachments use authenticated resumable chunks, a 512 MiB primary limit, a 256 KiB preview limit, and no bulk airtime carrier. | Andri (SEC/PROD/NET); external reviewer and field evaluator: **Unassigned** | Automated evidence | [ADR-0015](adr/0015-encrypted-attachment-pipeline.md); [protocol constants](../crates/kult-protocol/src/attachment.rs); [attachment e2e](../crates/kult-node/tests/attachments_e2e.rs); [media store tests](../crates/kult-store/tests/media.rs) | Accept ADR-0015 or successor; atomic transition coverage; protected-file device matrix; independent review | 2026-08-09 | -| **SV1-C08 Backup and recovery** | A versioned encrypted backup restores eligible user state on a clean supported device while live protocol secrets reset; offline account-root recovery replaces a lost device set. | Andri (SEC/PROD); external reviewer and field evaluator: **Unassigned** | Automated evidence for KKR7 round trips; stable authority designed | [Storage contract](07-storage.md#4-backup--portability); [backup tests](../crates/kult-store/tests/backup.rs); [backup e2e](../crates/kult-node/tests/backup_e2e.rs); [ADR-0026](adr/0026-revocable-device-authority.md) | Offline-root implementation, format freeze, conflict/stolen-device tests, clean-device field matrix, independent review | 2026-08-09 | +| **SV1-C05 Pairwise text bounds and atomicity** | One stable-v1 text event carries at most 65,507 UTF-8 bytes and changes visible/delivery state only through its complete atomic transition. | Andri (SEC/PROD); external reviewer: **Unassigned** | Test evidence | [Content codec](../crates/kult-protocol/src/content.rs); [atomic inventory](34-atomic-transition-inventory.md); [ADR-0028](adr/0028-atomic-protocol-commits.md); [typed commit plans](../crates/kult-store/src/commit.rs); [crash matrix](../crates/kult-node/src/atomic_tests.rs); [node e2e](../crates/kult-node/tests/node_e2e.rs) | Retained branch run, independent protocol review and supported-platform sudden-power-loss qualification | 2026-08-09 | +| **SV1-C06 Bounded groups** | Stable-v1 groups support at most 64 active accounts and authenticate the claimed sender separately for every recipient. | Andri (SEC/PROD); external reviewer: **Unassigned** | Test evidence for bounded group transactions; required origin property designed | [Atomic inventory](34-atomic-transition-inventory.md); [group crash matrix](../crates/kult-node/src/atomic_tests.rs); [group tests](../crates/kult-node/tests/groups_e2e.rs); [ADR-0029](adr/0029-recipient-authenticated-groups.md) | Retained branch run, recipient-origin implementation, upgrade path, malicious-member evidence and independent review | 2026-08-09 | +| **SV1-C07 Bounded attachments** | Consented attachments use authenticated resumable chunks, a 512 MiB primary limit, a 256 KiB preview limit, and no bulk airtime carrier. | Andri (SEC/PROD/NET); external reviewer and field evaluator: **Unassigned** | Test evidence for bounded metadata and file-first transitions | [Atomic inventory](34-atomic-transition-inventory.md); [ADR-0015](adr/0015-encrypted-attachment-pipeline.md); [protocol constants](../crates/kult-protocol/src/attachment.rs); [attachment e2e](../crates/kult-node/tests/attachments_e2e.rs); [media store tests](../crates/kult-store/tests/media.rs) | Retained branch run, ADR-0015 acceptance or successor, protected-file device matrix and independent review | 2026-08-09 | +| **SV1-C08 Backup and recovery** | A versioned encrypted backup restores eligible user state on a clean supported device while live protocol secrets reset; offline account-root recovery replaces a lost device set. | Andri (SEC/PROD); external reviewer and field evaluator: **Unassigned** | Test evidence for KKR7 round trips and atomic destination replacement; stable authority designed | [Atomic inventory](34-atomic-transition-inventory.md); [storage contract](07-storage.md#4-backup--portability); [backup tests](../crates/kult-store/tests/backup.rs); [backup e2e](../crates/kult-node/tests/backup_e2e.rs); [ADR-0026](adr/0026-revocable-device-authority.md) | Retained branch run, offline-root implementation, format freeze, conflict/stolen-device tests, clean-device field matrix and independent review | 2026-08-09 | | **SV1-C09 Blocking** | Blocking is an exact local identity rule that removes local relationship capabilities and state without claiming remote erasure or global identity revocation. | Andri (SEC/PROD); external abuse/usability evaluator: **Unassigned** | Designed | [ADR-0030](adr/0030-first-contact-admission.md#4-accept-reject-block-and-invite-are-explicit-state-transitions); [profile boundary](30-stable-v1-product-profile.md#7-blocking-and-deletion) | Implement block state/transitions across carriers, groups, capabilities, restore, and supported clients; adversarial/usability evidence | 2026-08-09 | | **SV1-C10 Honest delivery** | Queued means durable local custody, sent means bounded next-hop custody, and delivered requires an authenticated end-to-end receipt; none means read. | Andri (SEC/NET/PROD); external evaluator: **Unassigned** | Automated evidence for current ladder; stable custody designed | [Architecture lifecycle](03-architecture.md#3-message-lifecycle); [transport semantics](05-transports.md); [node e2e](../crates/kult-node/tests/node_e2e.rs); [ADR-0032](adr/0032-leased-mailbox-delivery.md) | Durable direct admission, mailbox v2, crash matrix, operator and real-network qualification | 2026-08-09 | | **SV1-C11 Supported platforms** | The stable release supports only the named device, OS, architecture, lifecycle, install, upgrade, and accessibility cells published as passed. | Andri (PROD/REL); independent field/accessibility evaluator: **Unassigned** | Implemented shells with automated build/simulator evidence | [Platform rule](30-stable-v1-product-profile.md#1-installation-and-supported-systems); [release gate](24-local-release-gate.md) | P0-09 named physical matrix and exact macOS/Linux support cells | 2026-08-09 | diff --git a/docs/34-atomic-transition-inventory.md b/docs/34-atomic-transition-inventory.md new file mode 100644 index 0000000..ac73d55 --- /dev/null +++ b/docs/34-atomic-transition-inventory.md @@ -0,0 +1,181 @@ +# Atomic protocol-transition inventory + +**Inventory date:** 2026-07-27 + +**Scope:** the implemented Alpha paths that overlap the frozen stable-v1 +profile, plus every adjacent persisted path found in the `kult-node` → +`kult-store` call graph + +**Disposition:** implementation and local test evidence; ADR-0028 remains +Proposed + +This inventory is the acceptance checklist for +[ADR-0028](adr/0028-atomic-protocol-commits.md). It distinguishes a complete +typed transition from an adjacent Alpha path that is not yet eligible for +stable-v1. A row marked excluded or open is not evidence of universal protocol +atomicity. + +## 1. Transaction contract + +The store exposes eleven bounded protocol plan kinds: + +| Plan | One logical transition | Principal bound | +|---|---|---| +| `PrekeyPublish` | Issue one fresh out-of-band one-time-prekey bundle and replace the exact vault that owns it | One exact vault replacement | +| `PairwiseSend` | Advance up to eight device sessions and retain every resulting ciphertext with its history, delivery, schedule, attachment, or control consequence | 8 sessions, 128 queue rows, 512 mutations | +| `PairwiseReceive` | Accept one pairwise plaintext/control and advance its receiving and optional receipt-sending state | 128 queue rows, 512 mutations | +| `HandshakeReceive` | Consume an optional one-time prekey and establish the exact session and accepted first-flight consequence | 8 device records, 128 queue rows, 512 mutations | +| `ReceiptReceive` | Accept one authenticated receipt/control, advance its session, and apply its exact delivery or deferred-work consequence | 128 queue rows, 512 mutations | +| `GroupSend` | Advance one sender chain or perform one late fan-out and retain all recipient-scoped copies and delivery rows | 64 accounts, 8 devices per account, 512 queue rows, 2,048 mutations | +| `GroupReceive` | Advance one receiver chain and retain the accepted plaintext consequence plus its encrypted receipt | One group chain and one receipt session | +| `GroupState` | Apply one roster, authority, announcement, receiver-chain, removal, or deferred group-control transition | 256 exact mutations | +| `AttachmentStage` | Create the bounded metadata graph for one outbound attachment manifest | 256 mutations | +| `AttachmentState` | Apply one bounded transfer/object/missing-range/deferred-control transition | 256 mutations | +| `Maintenance` | Apply one bounded retry, expiry, tombstone, terminal-input, repair, queue, or presentation acknowledgement transition | 256 exact mutations | + +Every plan validates its complete before-state before `BEGIN IMMEDIATE`. +`CommitPlan` is the only protocol-state write surface used by stable-profile +node modules. The source guard in `atomic_tests.rs` rejects direct session, +group, history, delivery, queue, replay, ephemeral, and media setters in those +modules. The one quarantined exception is the current linked-device Alpha +module described in section 4. + +The ownership rules are structural: + +- an advanced pairwise sending session owns at least one durable ciphertext, + and each retained message ciphertext has exactly one per-device delivery + owner; +- an advanced group sender chain owns one immutable group event and every + eligible account has exactly one logical delivery, with no more than eight + physical copies; +- an advanced receiver chain owns the accepted plaintext/control consequence, + replay marker, source-row acknowledgement, and any receipt ciphertext; +- a consumed one-time prekey owns the newly established exact-device session; + issued one-time prekeys become visible to the caller only after the + replacement vault commits; +- detached candidates replace live memory only after the commit receipt; +- a presentation marker commits with every visible change, so a restart after + commit but before event delivery requires a complete snapshot resync. + +Transport sends, discovery publication, native wake, call presentation, and UI +events occur after the database commit. File transfer uses a separate +file-first rule: a temporary authenticated chunk may reach the filesystem +before its metadata transition, but it is unreachable as accepted media until +that transition commits and restart reconciliation removes abandoned files. + +## 2. Stable-profile path inventory + +| Path | Advanced or destroyed material | Atomic owner | Restart and side-effect disposition | +|---|---|---|---| +| Fresh prekey-bundle export | One-time-prekey vault | `PrekeyPublish` | Bundle return follows commit; failure leaves the live and durable vault unchanged | +| Outbound first flight | New sending session, ciphertext, history and delivery | `PairwiseSend` | Session never exists without its queued ciphertext | +| Inbound first flight | Optional consumed one-time prekey, new session, accepted first content, receipt | `HandshakeReceive` | OPK removal and session establishment are one transaction | +| Pairwise text, edits and ordinary versioned content | Sending or receiving ratchet, immutable history, replay, receipts | `PairwiseSend` / `PairwiseReceive` | Presentation follows commit; duplicate input is absorbed | +| Pairwise capabilities and protocol controls, including role/admin requests | Pairwise ratchet and typed control consequence | Send/receive/receipt plans | Authenticated deferred work is durable before follow-up and deleted with that follow-up | +| Group create, invite acceptance, roster change and leave/removal | Group record, sender generation, receiver chains and contact stubs | `GroupState` | One bounded roster transition; events follow commit | +| Group chain announcement and acknowledgement | Pairwise session, pending announcement and receiver chain | `PairwiseSend`, `ReceiptReceive`, `GroupState` | Announcement ciphertext owns pending state; accepted chain owns exact deferred-control deletion | +| Group authority, roles, transfer and owner moderation | Signed authority record, generation and immutable announcement | `GroupSend`, `PairwiseSend`, `GroupState` | Authority state and its authenticated announcement/control consequence commit together | +| Group text, edits, polls and ephemeral events | Sender chain, immutable event, recipient/device deliveries and queue | `GroupSend` | At most 63 remote accounts × 8 devices = 504 physical copies in stable-v1 | +| Group receive | Receiver chain, accepted plaintext, replay state and receipt session/ciphertext | `GroupReceive` | Duplicate/reordered input does not advance the chain twice | +| Late group fan-out and partial carrier handoff | Retained ciphertext, new device deliveries and queue rows | `GroupSend` | Does not re-encrypt or advance the sender chain; restart retains unsent copies | +| Outbound attachment offer | Manifest history, transfer/object graph and optional view-once marker | `AttachmentStage`, then send plan | Manifest encryption waits for complete staged objects | +| Inbound attachment offer | Accepted manifest history, transfer/object graph, replay and receipt | Receive plan | No transfer becomes visible before the accepting receive commits | +| Attachment request, chunk, completion and refusal | Transfer/object progress, missing ranges and accepted deferred control | `AttachmentState` or response-owning `PairwiseSend` | The encrypted response and consumed request commit together when a response advances a session | +| Attachment expiry/view-once | Tombstone, plaintext history removal and media references | `Maintenance` | The tombstone and removals are one bounded transition; later input cannot revive plaintext | +| Scheduled-message activation | Schedule row, ratchet or sender chain, history, delivery and ciphertext | `PairwiseSend` / `GroupSend` | No transport or activation event occurs before commit; failed activation retains the schedule | +| Schedule create/edit/cancel | One sealed local outbox row | Single-row store operation | No cryptographic state, queue row or transport work exists before activation | +| Call-control send/receive | Pairwise ratchet and encrypted transient control | Pairwise send/receive plans | Signalling commits before in-memory call state or call events; live call/media state is intentionally process-local | +| Deferred inbox acceptance | Complete sealed carrier envelope | Bounded idempotent `pending_push` admission | Admission advances no cryptographic state; the consuming plan deletes the exact named row | +| Retry, expiry, terminal rejection and stale-session reset | Queue schedule/removal, delivery state, replay, session/capability reset | `Maintenance` | Work is paged at 256 mutations; retryable input remains durable | +| Event-delivery recovery | Sealed presentation marker | Visible plan plus `Maintenance` acknowledgement | Reopen emits `StateResyncRequired`; acknowledgement follows delivery | +| Media restart reconciliation | Missing-file object state and abandoned filesystem rows | Paged `AttachmentState` | Metadata repair commits before orphan cleanup; each page is bounded | +| Backup export | A read-only encrypted snapshot with fresh mnemonic | No live-state transition | Export excludes ratchets, prekeys, queues, live ephemeral plaintext/media and call state | +| Backup restore | New sibling database, reset markers, fresh device state and fresh prekeys | Sibling-store initialization plus atomic filesystem replacement | The destination is absent or a complete openable store; a visible restored identity never lacks fresh non-portable secrets | + +Edits, polls, roles and ephemeral content do not receive a special durability +exception: they are immutable authenticated content carried by the same +pairwise or group plans. Their convergent read projections run only after the +accepted event is durable. + +## 3. Store-call audit + +The production node call graph was searched for every raw `put`, `set`, +`update`, `delete`, queue, replay, seen, session, group, media and history +write. The remaining direct calls fall into these categories: + +| Remaining direct write | Classification | +|---|---| +| Labels, folders, pins, icons, theme, petnames and note-to-self | Local sealed presentation/organization state; no ratchet, sender chain, replay, delivery or carrier consequence | +| Schedule create/edit/cancel | One local row; activation is typed | +| Complete-envelope `pending_push` | Bounded idempotent ingress staging before parsing or cryptographic work | +| Media garbage collection after semantic commit | Physical cleanup after the durable tombstone/progress transition | +| Contact import, hint/verification changes | Current automatic-contact Alpha flow; outside stable-v1 pending ADR-0030 | +| `devices.rs` authority, linking and sync setters | Current ADR-0024 Alpha authority; outside stable-v1 pending ADR-0026 | + +No stable-profile sender/receiver chain, one-time prekey consumption, group +state, protocol history, delivery, outbound queue, replay/seen, attachment +state, or ephemeral tombstone is written through those local-state calls. +Adding such a call to a stable protocol module fails the source guard. + +## 4. Open and excluded paths + +These boundaries keep ADR-0028 Proposed: + +1. **Linked-device authority and sync are an open P0 path, not a P2 + exception.** The current ADR-0024 implementation copies the account root and + applies linking, manifest, counter, convergence-log and projection writes + through the quarantined `devices.rs` path. The stable-v1 authority required + by ADR-0026 does not exist yet, so there is no honest stable transition to + freeze. ADR-0026 must define its typed manifest, root-recovery, link-package + and sync-import plans and add them to this matrix. +2. **First-contact admission is an open P0 path.** Current explicit contact + import can update the contact and endpoint rows before the future ADR-0030 + consent/quarantine transition exists. The current inbound cryptographic + handshake itself is atomic, but the automatic-contact Alpha policy is + outside stable-v1. +3. **Initial profile creation is a bootstrap boundary.** It creates no prior + live state and returns no usable node until identity, device state and + prekeys all exist, but an interrupted initial create can leave an unusable + partial profile path. Crash-safe sibling creation remains required before + ADR-0028 acceptance. +4. **Live call state is process-local by design.** Ratchet-protected signalling + is covered; ringing, active-call and media state are not restored after a + process stop and are not stable persisted state. +5. **Mailbox-v1 relay custody remains outside this node/store transaction.** + Endpoint inbox acceptance is covered, but leased relay deletion only after + endpoint acknowledgement remains proposed in ADR-0032. + +The intentionally excluded P2 paths are live video, groups above 64 accounts, +advanced moderation, high-bandwidth media, Freenet-style or other additional +delay-tolerant carriers, cross-protocol federation, richer optional +discovery/wake services, and later governance expansion. None is evidence for +or against stable-v1 atomicity. + +## 5. Failure and restart matrix + +`crates/kult-node/src/atomic_tests.rs` applies every transaction failpoint to +all eleven plan kinds: + +- before and after `BEGIN IMMEDIATE`; +- before and after every numbered logical statement; +- before and after commit; +- before and after candidate cryptography and memory replacement; +- before and after event delivery; and +- disk-full, constraint and duplicate-index failure classes. + +The same suite covers duplicate and reordered deferred input, retry after +restart, presentation-outbox recovery, scheduled activation, and a maximum +stable-v1 group fan-out of 504 physical deliveries. The group end-to-end suite +covers partial carrier handoff followed by sender restart. Media tests cover +duplicate chunks, interrupted temporary files, exact missing ranges and full +quotas. Backup tests inject every atomic-replacement phase and initializer +failure. + +For each injected point, reopen observes either the complete transition or its +complete absence. The input remains retryable in the absent case; the durable +case absorbs replay and requests presentation resynchronization when needed. +No test accepts an intermediate chain/session state. + +This repository evidence is not physical sudden-power-loss qualification, +external review, or independently produced interoperability evidence. Those +remain P0 gates in the [release evidence ledger](31-release-evidence-ledger.md). diff --git a/docs/adr/0028-atomic-protocol-commits.md b/docs/adr/0028-atomic-protocol-commits.md index 854642a..e08878d 100644 --- a/docs/adr/0028-atomic-protocol-commits.md +++ b/docs/adr/0028-atomic-protocol-commits.md @@ -1,7 +1,7 @@ # ADR-0028: Atomic protocol-state commits -- **Status**: Proposed; pairwise commit-plan slice implemented -- **Date**: 2026-07-26 +- **Status**: Proposed; stable-profile implementation inventory recorded +- **Date**: 2026-07-27 ## Context @@ -11,8 +11,7 @@ advanced cryptographic state in separate SQLite autocommits from message history, replay markers, receipt routes, delivery rows, and outbound queue entries. A process stop or disk error between writes could make a valid inbound ciphertext permanently undecryptable or advance an outbound chain without -retaining the only ciphertext produced by that step. Group sender/receiver -transitions still need the corresponding typed plans. +retaining the only ciphertext produced by that step. Deferred inbox rows also need at-least-once processing: removing a row before all durable consequences commit turns an ordinary crash into message loss. @@ -31,9 +30,13 @@ work only on those candidate values and creates one bounded typed commit plan: - `PairwiseSend`; - `PairwiseReceive`; +- `PrekeyPublish`; - `HandshakeReceive`; - `GroupSend`; - `GroupReceive`; +- `GroupState`; +- `AttachmentStage`; +- `AttachmentState`; - `ReceiptReceive`; or - `Maintenance`. @@ -142,43 +145,50 @@ or plaintext consequence. ### 7. Implementation and evidence status -The implemented pairwise slice now provides bounded `PairwiseSend`, -`PairwiseReceive`, `HandshakeReceive`, `ReceiptReceive`, and `Maintenance` -plans. It covers ordinary and versioned text, edits, pairwise attachment offers, -ephemeral content, capability controls, receipts, scheduled activation, -late-device delivery, bounded unconfirmed-session repair, reset re-handshakes, -retry/expiry and view-once-tombstone maintenance, and exact pending-row -acknowledgement. Sessions and the prekey vault are cloned before cryptographic -work; live memory changes only after a successful commit receipt. +The implementation now provides all eleven plan kinds above. They cover +pairwise and group text, edits, polls, roles, authority changes, group +announcements and bounded fan-out, pairwise and group attachments, missing +ranges, ephemeral/view-once state, scheduled activation, call signalling, +late-device delivery, exact deferred-control acknowledgement, retry/expiry, +session repair, media reconciliation, and presentation recovery. Fresh +out-of-band one-time-prekey issuance uses `PrekeyPublish`; inbound consumption +uses `HandshakeReceive` and cannot commit without the established session. Each plan validates its bounded before/after relationships before -`BEGIN IMMEDIATE`. The transaction then writes the advanced candidate state and -every paired ciphertext or accepted plaintext consequence. Complete fresh -envelopes enter the durable pending inbox before parsing or cryptographic work. -Top-level fragments are the bounded exception: assembly advances no ratchet, -the completed inner envelope is staged before deferral, and refused assembly -leaves the fragments unseen for carrier retry. +`BEGIN IMMEDIATE`. The transaction writes the detached candidate and every +paired ciphertext or accepted plaintext consequence. Complete fresh envelopes +enter the durable pending inbox before parsing or cryptographic work. Top-level +fragments are the bounded exception: assembly advances no ratchet, the +completed inner envelope is staged before deferral, and refused assembly leaves +the fragments unseen for carrier retry. A sealed presentation marker is written in the same transaction as every -pairwise-visible transition. If the process stops after commit but before event +visible transition. If the process stops after commit but before event delivery, reopening the node emits `StateResyncRequired`; the marker is removed only after a later tick acknowledges delivery. The FFI and daemon surfaces preserve that signal, and the desktop shell responds by re-reading its visible -snapshots. The deterministic crash suite in -`crates/kult-node/src/atomic_tests.rs` exercises before/after cryptographic -steps, every logical transaction statement, commit, memory replacement, event -delivery, disk-full, constraint, duplicate, reorder, deferred-input, expiry, -session-repair, and restart cases for the five implemented plan kinds. - -This is not full ADR acceptance. `GroupSend` and `GroupReceive` plans remain -unimplemented. Authenticated attachment and group controls are durably accepted -before their bounded follow-up runs, but their group/media follow-up mutations -still need a single typed plan with the deferred-control acknowledgement. -Real-time call state remains intentionally process-local and is not restored -after a restart. Unknown valid handshakes still enter the ordinary contact table -rather than the ADR-0030 request quarantine. Those gaps keep this ADR Proposed -and prevent the pairwise evidence from being presented as universal protocol -atomicity. +snapshots. + +The deterministic crash suite in `crates/kult-node/src/atomic_tests.rs` +exercises before/after cryptographic steps, every logical transaction +statement, commit, memory replacement, event delivery, disk-full, constraint, +duplicate, reorder, deferred-input, expiry, session-repair, scheduled +activation, maximum stable-v1 group fan-out, and restart cases for every plan +kind. Group end-to-end evidence covers partial carrier handoff and restart; +media evidence covers duplicate chunks, interrupted files and full quotas; +backup evidence covers each replacement phase and fresh-secret initialization. +The complete path-by-path disposition is the +[atomic transition inventory](../34-atomic-transition-inventory.md). + +This is not full ADR acceptance. The current linked-device Alpha implementation +uses the authority design that ADR-0026 must replace and retains quarantined +multi-write link/sync paths. The current automatic-contact flow remains outside +stable-v1 pending ADR-0030. Interrupted initial profile creation can leave an +unusable partial path, mailbox-v1 cannot yet acknowledge leased relay custody +after endpoint commit, and live call state remains intentionally process-local. +Independent review and supported-platform sudden-power-loss qualification are +also absent. These gaps keep this ADR Proposed and prevent the implemented +matrix from being presented as universal protocol atomicity. ## Alternatives considered diff --git a/docs/adr/README.md b/docs/adr/README.md index 977e23c..6c24cfd 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -34,7 +34,7 @@ product promise merely because the file exists. | [0025](0025-optional-freenet-carrier.md) | Proposed | Optional epoch-scoped Freenet store-and-forward carrier and metadata boundary | | [0026](0026-revocable-device-authority.md) | Proposed (P0) | Required offline account-root recovery and quorum-authorized manifests replacing ADR-0024 authority before stable | | [0027](0027-opaque-indexed-store.md) | Accepted; implemented | Versioned keyed indexes, row-bound sealing, transactional migration, and honest local deletion limits | -| [0028](0028-atomic-protocol-commits.md) | Proposed; pairwise plans implemented | Transactional ratchet, handshake, group-chain, queue, replay, and deferred-inbox state transitions | +| [0028](0028-atomic-protocol-commits.md) | Proposed; stable-profile inventory implemented with open boundaries | Transactional ratchet, handshake, group-chain, queue, replay, attachment, maintenance, and deferred-inbox state transitions | | [0029](0029-recipient-authenticated-groups.md) | Proposed | Per-recipient origin authentication for sender-key group messages and author-sensitive state | | [0030](0030-first-contact-admission.md) | Proposed | Bounded public introductions, provisional message requests, blocking, and group-invite consent | | [0031](0031-capability-scoped-dht-discovery.md) | Proposed | Capability-scoped, encrypted DHT first contact with mailbox-only public routes | From df0d2dd992fb4cd60cfd2c4a4135af01c504e46e Mon Sep 17 00:00:00 2001 From: Andri Date: Mon, 27 Jul 2026 16:06:33 +0000 Subject: [PATCH 2/2] Link ADR-0028 retained verification --- docs/31-release-evidence-ledger.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/31-release-evidence-ledger.md b/docs/31-release-evidence-ledger.md index f9a29d6..f47b2d4 100644 --- a/docs/31-release-evidence-ledger.md +++ b/docs/31-release-evidence-ledger.md @@ -36,7 +36,7 @@ accepts it and the assignment is recorded in |---|---|---|---|---|---| | **P0-01 Honest claims and evidence ledger** | Andri (FND; interim SEC/PROD) | Designed: vocabulary, frozen claim register, and this ledger exist; the repository documentation check passed in release-control CI. | **Open.** External website and repository-description corrections remain; no stable release evidence bundle exists. | [Stabilization vocabulary](29-stabilization-program.md#2-evidence-vocabulary); [stable-v1 profile](30-stable-v1-product-profile.md); [CI run 199](https://github.com/AndriGitDev/Komms/actions/runs/30202463092); this ledger; [public-copy follow-up](#4-public-copy-audit) | 2026-08-09 | | **P0-02 Name-risk assessment and recorded decision** | Andri (FND and project risk owner); qualified trademark counsel: **Unassigned** | Designed: dated founder decision, observed overlap, migration cost, cadence, and escalation triggers recorded. | **Open.** This is not legal clearance. No qualified similarity/class/jurisdiction opinion or trademark/asset policy exists. | [Name-risk decision](32-name-risk-decision.md); [brand system](28-brand-system.md) | 2026-10-26, or trigger event | -| **P0-03 Stabilized core product profile** | Andri (FND; interim PROD/SEC) | Designed: product boundary, bounds, supported-system rule, services, and exclusions frozen. Typed pairwise, group, attachment, scheduled-activation and bounded-maintenance plans have deterministic local crash/restart tests. | **Open.** ADR-0026 and ADR-0028 through ADR-0032 remain proposed. Linked-device authority/sync, first-contact admission, initial profile creation and leased relay custody remain outside the complete plan matrix; no retained branch run, field qualification or independent review has closed those gaps. | [Stable-v1 profile](30-stable-v1-product-profile.md); [atomic inventory](34-atomic-transition-inventory.md); [ADR-0028](adr/0028-atomic-protocol-commits.md); [commit plans](../crates/kult-store/src/commit.rs); [crash matrix](../crates/kult-node/src/atomic_tests.rs); [P0 ADR index](adr/README.md) | 2026-08-09 | +| **P0-03 Stabilized core product profile** | Andri (FND; interim PROD/SEC) | Designed: product boundary, bounds, supported-system rule, services, and exclusions frozen. Typed pairwise, group, attachment, scheduled-activation and bounded-maintenance plans have deterministic crash/restart tests retained by CI run 214. | **Open.** ADR-0026 and ADR-0028 through ADR-0032 remain proposed. Linked-device authority/sync, first-contact admission, initial profile creation and leased relay custody remain outside the complete plan matrix; field qualification and independent review have not closed those gaps. | [Stable-v1 profile](30-stable-v1-product-profile.md); [atomic inventory](34-atomic-transition-inventory.md); [ADR-0028](adr/0028-atomic-protocol-commits.md); [commit plans](../crates/kult-store/src/commit.rs); [crash matrix](../crates/kult-node/src/atomic_tests.rs); [branch revision](https://github.com/AndriGitDev/Komms/commit/f17a60636a074b889105d7e123caf0fa475bebfc); [CI run 214](https://github.com/AndriGitDev/Komms/actions/runs/30281155361); [P0 ADR index](adr/README.md) | 2026-08-09 | | **P0-04 Clean-install and real-network golden path** | Andri (interim NET/PROD); independent field evaluator: **Unassigned** | Implemented with local/CI evidence for internet components and shells. | **Open.** No qualified default bootstrap/mailbox, clean-device distinct-NAT matrix, default-blackhole journey, replacement operator, or pure-core journey. | [Internet tests](../crates/kult-node/tests/internet_e2e.rs); [Alpha guide](27-alpha-testing.md); [ADR-0034](adr/0034-operator-minimized-reference-discovery.md) | 2026-08-09 | | **P0-05 Unsolicited-contact abuse admission** | Andri (interim SEC/NET/PROD); independent adversarial/usability evaluator: **Unassigned** | Designed, with automated evidence for interim byte/count bounds. | **Open.** No pre-KEM admission, durable request inbox, consent promotion, identity block implementation, or adversarial/field evidence. Current first contact can create normal contact state. | [ADR-0030](adr/0030-first-contact-admission.md); [transport bounds](05-transports.md); [protocol tests](../crates/kult-protocol/tests/protocol.rs) | 2026-08-09 | | **P0-06 Independent crypto and protocol assurance** | Andri (interim SEC); independent cryptography reviewer: **Unassigned**; independent interoperability implementer: **Unassigned** | Automated evidence for local KATs, properties, sessions, and protocol decoding. | **Open.** No external vectors, separate implementation, review scope, findings report, disposition, or residual-risk statement. P0 protocol/security ADRs remain proposed. | [Baseline CI](https://github.com/AndriGitDev/Komms/actions/runs/30199264838); [Crypto KATs](../crates/kult-crypto/tests/kat.rs); [properties](../crates/kult-crypto/tests/properties.rs); [session tests](../crates/kult-crypto/tests/session.rs); [cryptography spec](04-cryptography.md) | 2026-08-09 | @@ -58,10 +58,10 @@ the listed gap. A new stable claim requires a new identifier here. | **SV1-C02 Local identity** | Komms identity can be created without a required phone number, email address, real name, or project account. | Andri (SEC/PROD); external reviewer: **Unassigned** | Automated evidence | [Identity design](06-identity-trust.md); [node e2e](../crates/kult-node/tests/node_e2e.rs); [FFI e2e](../crates/kult-ffi/tests/ffi_e2e.rs) | Independent protocol review and supported-platform clean-install evidence | 2026-08-09 | | **SV1-C03 Pairwise confidentiality and authenticity** | Accepted contacts exchange authenticated end-to-end encrypted pairwise text; intermediaries receive sealed envelopes rather than message plaintext. | Andri (SEC); external cryptography reviewer: **Unassigned** | Test evidence | [`main@4fda544`](https://github.com/AndriGitDev/Komms/tree/4fda544739c0665b6a324256d858c16c1d73d992); [KATs](../crates/kult-crypto/tests/kat.rs); [node e2e](../crates/kult-node/tests/node_e2e.rs); [pairwise crash matrix](../crates/kult-node/src/atomic_tests.rs) | P0-06 external vectors, review, and interoperability evidence | 2026-08-09 | | **SV1-C04 Contact establishment** | Intentional Connect codes and bounded message requests establish contact without making a project service the identity authority. | Andri (SEC/NET/PROD); external evaluators: **Unassigned** | Designed | [ADR-0030](adr/0030-first-contact-admission.md); [ADR-0031](adr/0031-capability-scoped-dht-discovery.md) | Implement, accept, adversarially test, and field-qualify v2 contact flow | 2026-08-09 | -| **SV1-C05 Pairwise text bounds and atomicity** | One stable-v1 text event carries at most 65,507 UTF-8 bytes and changes visible/delivery state only through its complete atomic transition. | Andri (SEC/PROD); external reviewer: **Unassigned** | Test evidence | [Content codec](../crates/kult-protocol/src/content.rs); [atomic inventory](34-atomic-transition-inventory.md); [ADR-0028](adr/0028-atomic-protocol-commits.md); [typed commit plans](../crates/kult-store/src/commit.rs); [crash matrix](../crates/kult-node/src/atomic_tests.rs); [node e2e](../crates/kult-node/tests/node_e2e.rs) | Retained branch run, independent protocol review and supported-platform sudden-power-loss qualification | 2026-08-09 | -| **SV1-C06 Bounded groups** | Stable-v1 groups support at most 64 active accounts and authenticate the claimed sender separately for every recipient. | Andri (SEC/PROD); external reviewer: **Unassigned** | Test evidence for bounded group transactions; required origin property designed | [Atomic inventory](34-atomic-transition-inventory.md); [group crash matrix](../crates/kult-node/src/atomic_tests.rs); [group tests](../crates/kult-node/tests/groups_e2e.rs); [ADR-0029](adr/0029-recipient-authenticated-groups.md) | Retained branch run, recipient-origin implementation, upgrade path, malicious-member evidence and independent review | 2026-08-09 | -| **SV1-C07 Bounded attachments** | Consented attachments use authenticated resumable chunks, a 512 MiB primary limit, a 256 KiB preview limit, and no bulk airtime carrier. | Andri (SEC/PROD/NET); external reviewer and field evaluator: **Unassigned** | Test evidence for bounded metadata and file-first transitions | [Atomic inventory](34-atomic-transition-inventory.md); [ADR-0015](adr/0015-encrypted-attachment-pipeline.md); [protocol constants](../crates/kult-protocol/src/attachment.rs); [attachment e2e](../crates/kult-node/tests/attachments_e2e.rs); [media store tests](../crates/kult-store/tests/media.rs) | Retained branch run, ADR-0015 acceptance or successor, protected-file device matrix and independent review | 2026-08-09 | -| **SV1-C08 Backup and recovery** | A versioned encrypted backup restores eligible user state on a clean supported device while live protocol secrets reset; offline account-root recovery replaces a lost device set. | Andri (SEC/PROD); external reviewer and field evaluator: **Unassigned** | Test evidence for KKR7 round trips and atomic destination replacement; stable authority designed | [Atomic inventory](34-atomic-transition-inventory.md); [storage contract](07-storage.md#4-backup--portability); [backup tests](../crates/kult-store/tests/backup.rs); [backup e2e](../crates/kult-node/tests/backup_e2e.rs); [ADR-0026](adr/0026-revocable-device-authority.md) | Retained branch run, offline-root implementation, format freeze, conflict/stolen-device tests, clean-device field matrix and independent review | 2026-08-09 | +| **SV1-C05 Pairwise text bounds and atomicity** | One stable-v1 text event carries at most 65,507 UTF-8 bytes and changes visible/delivery state only through its complete atomic transition. | Andri (SEC/PROD); external reviewer: **Unassigned** | Test evidence | [Content codec](../crates/kult-protocol/src/content.rs); [atomic inventory](34-atomic-transition-inventory.md); [ADR-0028](adr/0028-atomic-protocol-commits.md); [typed commit plans](../crates/kult-store/src/commit.rs); [crash matrix](../crates/kult-node/src/atomic_tests.rs); [node e2e](../crates/kult-node/tests/node_e2e.rs); [CI run 214](https://github.com/AndriGitDev/Komms/actions/runs/30281155361) | Independent protocol review and supported-platform sudden-power-loss qualification | 2026-08-09 | +| **SV1-C06 Bounded groups** | Stable-v1 groups support at most 64 active accounts and authenticate the claimed sender separately for every recipient. | Andri (SEC/PROD); external reviewer: **Unassigned** | Test evidence for bounded group transactions; required origin property designed | [Atomic inventory](34-atomic-transition-inventory.md); [group crash matrix](../crates/kult-node/src/atomic_tests.rs); [group tests](../crates/kult-node/tests/groups_e2e.rs); [ADR-0029](adr/0029-recipient-authenticated-groups.md); [CI run 214](https://github.com/AndriGitDev/Komms/actions/runs/30281155361) | Recipient-origin implementation, upgrade path, malicious-member evidence and independent review | 2026-08-09 | +| **SV1-C07 Bounded attachments** | Consented attachments use authenticated resumable chunks, a 512 MiB primary limit, a 256 KiB preview limit, and no bulk airtime carrier. | Andri (SEC/PROD/NET); external reviewer and field evaluator: **Unassigned** | Test evidence for bounded metadata and file-first transitions | [Atomic inventory](34-atomic-transition-inventory.md); [ADR-0015](adr/0015-encrypted-attachment-pipeline.md); [protocol constants](../crates/kult-protocol/src/attachment.rs); [attachment e2e](../crates/kult-node/tests/attachments_e2e.rs); [media store tests](../crates/kult-store/tests/media.rs); [CI run 214](https://github.com/AndriGitDev/Komms/actions/runs/30281155361) | ADR-0015 acceptance or successor, protected-file device matrix and independent review | 2026-08-09 | +| **SV1-C08 Backup and recovery** | A versioned encrypted backup restores eligible user state on a clean supported device while live protocol secrets reset; offline account-root recovery replaces a lost device set. | Andri (SEC/PROD); external reviewer and field evaluator: **Unassigned** | Test evidence for KKR7 round trips and atomic destination replacement; stable authority designed | [Atomic inventory](34-atomic-transition-inventory.md); [storage contract](07-storage.md#4-backup--portability); [backup tests](../crates/kult-store/tests/backup.rs); [backup e2e](../crates/kult-node/tests/backup_e2e.rs); [ADR-0026](adr/0026-revocable-device-authority.md); [CI run 214](https://github.com/AndriGitDev/Komms/actions/runs/30281155361) | Offline-root implementation, format freeze, conflict/stolen-device tests, clean-device field matrix and independent review | 2026-08-09 | | **SV1-C09 Blocking** | Blocking is an exact local identity rule that removes local relationship capabilities and state without claiming remote erasure or global identity revocation. | Andri (SEC/PROD); external abuse/usability evaluator: **Unassigned** | Designed | [ADR-0030](adr/0030-first-contact-admission.md#4-accept-reject-block-and-invite-are-explicit-state-transitions); [profile boundary](30-stable-v1-product-profile.md#7-blocking-and-deletion) | Implement block state/transitions across carriers, groups, capabilities, restore, and supported clients; adversarial/usability evidence | 2026-08-09 | | **SV1-C10 Honest delivery** | Queued means durable local custody, sent means bounded next-hop custody, and delivered requires an authenticated end-to-end receipt; none means read. | Andri (SEC/NET/PROD); external evaluator: **Unassigned** | Automated evidence for current ladder; stable custody designed | [Architecture lifecycle](03-architecture.md#3-message-lifecycle); [transport semantics](05-transports.md); [node e2e](../crates/kult-node/tests/node_e2e.rs); [ADR-0032](adr/0032-leased-mailbox-delivery.md) | Durable direct admission, mailbox v2, crash matrix, operator and real-network qualification | 2026-08-09 | | **SV1-C11 Supported platforms** | The stable release supports only the named device, OS, architecture, lifecycle, install, upgrade, and accessibility cells published as passed. | Andri (PROD/REL); independent field/accessibility evaluator: **Unassigned** | Implemented shells with automated build/simulator evidence | [Platform rule](30-stable-v1-product-profile.md#1-installation-and-supported-systems); [release gate](24-local-release-gate.md) | P0-09 named physical matrix and exact macOS/Linux support cells | 2026-08-09 |