diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index d34477c..2c50c2b 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -42,6 +42,18 @@ jobs: - name: tests run: cargo test --workspace --all-features + opaque-store-scale: + name: opaque storage scale budgets + runs-on: ubuntu-latest + env: + RUSTFLAGS: "-D warnings" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: 100k and 1m messages + run: scripts/store-scale-gate.sh + # Informational only: a failing coverage run must never block the # advisory signal this workflow exists for. coverage: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7b23b5..b9a34f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,25 @@ jobs: - name: cargo check run: cargo check --workspace --all-features --all-targets + storage-windows: + name: opaque storage on Windows + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: qualify storage host + shell: pwsh + run: | + $volume = Get-Volume -DriveLetter (Get-Location).Drive.Name + if ($volume.FileSystemType -ne "NTFS") { + throw "Storage tests require an explicitly identified NTFS volume" + } + Write-Output "OS: $([System.Runtime.InteropServices.RuntimeInformation]::OSDescription)" + Write-Output "Filesystem: $($volume.FileSystemType)" + - name: kult-store tests + run: cargo test -p kult-store --all-features -- --test-threads=1 + desktop: name: desktop app (fmt, clippy, tests) runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index a1d563f..15e8e98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -194,7 +194,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix", + "rustix 1.1.4", "slab", "windows-sys 0.61.2", ] @@ -229,6 +229,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atomicwrites" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef1bb8d1b645fe38d51dfc331d720fb5fc2c94b440c76cc79c80ff265ca33e3" +dependencies = [ + "rustix 0.38.44", + "tempfile", + "windows-sys 0.52.0", +] + [[package]] name = "attohttpc" version = "0.30.1" @@ -1755,6 +1766,7 @@ dependencies = [ name = "kult-store" version = "0.3.0" dependencies = [ + "atomicwrites", "fs2", "kult-crypto", "kult-protocol", @@ -2289,6 +2301,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2811,7 +2829,7 @@ dependencies = [ "concurrent-queue", "hermit-abi", "pin-project-lite", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -3276,6 +3294,19 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -3285,7 +3316,7 @@ dependencies = [ "bitflags 2.13.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -3691,7 +3722,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] diff --git a/README.md b/README.md index 5a74f80..afc088e 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ independent interoperability. | Area | Current state | |---|---| -| **Core security and storage** | Hybrid PQXDH, Double Ratchet sessions, sealed envelopes, sealed local record bodies, backup/recovery, RPC/CLI, and UniFFI paths are implemented with automated tests. The current SQLite schema still exposes exact contact/group identifiers in plaintext indexes and lacks universal row binding; ADR-0027 is required before stronger locked-database metadata claims. The combined implementation has not yet passed independent security review or independent interoperability gates. | +| **Core security and storage** | Hybrid PQXDH, Double Ratchet sessions, sealed envelopes, opaque keyed SQLite indexes, row-bound local records, released-schema migration, backup/recovery, RPC/CLI, and UniFFI paths are implemented with repeatable tests. SQLite still reveals approximate row counts/sizes, order, within-domain equality, access patterns, and change timing. Storage has Linux/ext4 test evidence, but independent review and physical macOS, Windows, Android, iOS, power-loss, backup-exclusion, and forensic qualification remain open. | | **Internet, LAN, and delayed delivery** | libp2p QUIC/TCP, Kademlia discovery, NAT traversal, mDNS, and volunteer mailbox roles are implemented. Fresh app installs do not yet have a qualified distinct-NAT golden path: bootstrap and mailbox defaults require deliberate configuration, and mailbox persistence/operator behavior remains a stabilization gate. [ADR-0034](docs/adr/0034-operator-minimized-reference-discovery.md) proposes an initial founder-operated Hetzner Standard-mode bootstrap/DHT/rendezvous default with RAM-backed mutable state; it is not implemented or a durable mailbox. | | **Off-grid delivery** | Sneakernet and the Meshtastic carrier, duty-cycle controls, retransmission, and internet↔mesh bridge paths are implemented with automated evidence. The physical two-radio bench is not yet field-qualified. | | **Applications and messaging** | Desktop, Android, and iOS shells expose pairwise/group text and a broad Alpha feature set, including attachments, local organization, linked devices, ephemeral content, polls, roles, and direct audio-call paths. CI and simulator evidence exist; hands-on device, background lifecycle, NAT, accessibility, and localization qualification remain. | diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lock index b92932a..b4331f0 100644 --- a/apps/desktop/src-tauri/Cargo.lock +++ b/apps/desktop/src-tauri/Cargo.lock @@ -212,7 +212,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix", + "rustix 1.1.4", "slab", "windows-sys 0.61.2", ] @@ -270,6 +270,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atomicwrites" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef1bb8d1b645fe38d51dfc331d720fb5fc2c94b440c76cc79c80ff265ca33e3" +dependencies = [ + "rustix 0.38.44", + "tempfile", + "windows-sys 0.52.0", +] + [[package]] name = "attohttpc" version = "0.30.1" @@ -2641,9 +2652,11 @@ dependencies = [ name = "kult-store" version = "0.3.0" dependencies = [ + "atomicwrites", "fs2", "kult-crypto", "kult-protocol", + "libc", "postcard", "rand_core 0.6.4", "rusqlite", @@ -3199,6 +3212,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -4108,7 +4127,7 @@ dependencies = [ "concurrent-queue", "hermit-abi", "pin-project-lite", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -4720,6 +4739,19 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -4729,7 +4761,7 @@ dependencies = [ "bitflags 2.13.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -5751,7 +5783,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] diff --git a/crates/kult-store/Cargo.toml b/crates/kult-store/Cargo.toml index 41a47a1..3eff8ab 100644 --- a/crates/kult-store/Cargo.toml +++ b/crates/kult-store/Cargo.toml @@ -11,13 +11,16 @@ repository.workspace = true kult-crypto = { version = "0.3.0", path = "../kult-crypto" } kult-protocol = { version = "0.3.0", path = "../kult-protocol" } rusqlite = { version = "0.31", features = ["bundled"] } -rand_core = "0.6" +rand_core = { version = "0.6", features = ["getrandom"] } serde = { version = "1", features = ["derive"] } postcard = { version = "1", default-features = false, features = ["alloc"] } zeroize = "1" fs2 = "0.4" libc = "0.2" +[target.'cfg(any(unix, windows))'.dependencies] +atomicwrites = "0.4.4" + [dev-dependencies] proptest = "1" rand = "0.8" diff --git a/crates/kult-store/src/backup.rs b/crates/kult-store/src/backup.rs index 193ed77..5fe0731 100644 --- a/crates/kult-store/src/backup.rs +++ b/crates/kult-store/src/backup.rs @@ -39,8 +39,12 @@ //! corrupted file are deliberately indistinguishable — uniform AEAD //! failure, no oracle. +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; + use rand_core::CryptoRngCore; -use rusqlite::params; +use rusqlite::Connection; use serde::{Deserialize, Serialize}; use zeroize::{Zeroize, Zeroizing}; @@ -51,10 +55,11 @@ use kult_crypto::{ use kult_protocol::DeviceSyncEvent; use crate::{ - ContactDeviceRecord, ContactRecord, DeviceStateRecord, EphemeralConversation, EphemeralRecord, - EphemeralState, GroupAuthorityRecord, GroupMember, GroupMessageRecord, GroupRecord, - LocalMetadataRecord, MessageRecord, NoteMessageRecord, PendingAnnounce, Result, Store, - StoreError, + acquire_database_identity_lock, acquire_store_lock, decode_exact as decode_store_exact, + migration, store_v2, ContactDeviceRecord, ContactRecord, DeviceStateRecord, + EphemeralConversation, EphemeralRecord, EphemeralState, GroupAuthorityRecord, GroupMember, + GroupMessageRecord, GroupRecord, LocalMetadataRecord, MessageRecord, NoteMessageRecord, + PendingAnnounce, Result, Store, StoreError, }; /// Backup file magic: Komms recovery file, format 7 (linked-device authority). @@ -74,6 +79,10 @@ pub const BACKUP_MAGIC_V1: [u8; 4] = *b"KKR1"; const BACKUP_AD: &[u8] = b"KK-backup-v1"; const HEADER_LEN: usize = 4 + 12 + 16; +const MAX_BACKUP_FILE_BYTES: u64 = 16 * 1024 * 1024 * 1024; +const MAX_BACKUP_RECORDS: u64 = 50_000_000; +const RESTORE_SPACE_RESERVE: u64 = 64 * 1024 * 1024; +const ESTIMATED_RESTORED_ROW_BYTES: u64 = 2_048; /// A group's durable identity in a backup: everything but the chains. #[derive(Serialize, Deserialize)] @@ -337,7 +346,9 @@ impl Store { profile: KdfProfile, rng: &mut impl CryptoRngCore, ) -> Result { - if backup.len() <= HEADER_LEN { + if backup.len() <= HEADER_LEN + || u64::try_from(backup.len()).map_or(true, |length| length > MAX_BACKUP_FILE_BYTES) + { return Err(StoreError::NotABackup); } let version = match <[u8; 4]>::try_from(&backup[..4]).expect("length checked") { @@ -364,7 +375,7 @@ impl Store { let kek = derive_kek(&entropy[..], &salt, file_profile)?; let key = StorageKey::from_bytes(*kek); let plain = Zeroizing::new(key.open(BACKUP_AD, &backup[HEADER_LEN..])?); - let mut payload: BackupPayload = match version { + let payload: BackupPayload = match version { 1 => { let v1: BackupPayloadV1 = decode_exact(&plain)?; BackupPayload { @@ -488,139 +499,65 @@ impl Store { 7 => decode_exact(&plain)?, _ => unreachable!("version matched above"), }; - let identity_bytes: Zeroizing<[u8; 64]> = Zeroizing::new( - payload.identity[..] - .try_into() - .map_err(|_| StoreError::NotABackup)?, - ); - payload.identity.zeroize(); - let identity = Identity::from_bytes(&identity_bytes); - let me = identity.public().ed; - - let store = Store::create(path, passphrase, profile, rng)?; - store.put_identity(&identity, rng)?; - for contact in &payload.contacts { - store.put_contact(contact, rng)?; - } - for endpoint in &payload.contact_devices { - store.put_contact_device(endpoint, rng)?; - } - for message in &payload.messages { - store.put_message(message, rng)?; - } - for peer in &payload.reset_peers { - store.put_reset_marker(peer)?; - } - for group in payload.groups { - // Fresh chain, announced to the full roster: the old chains died - // with the old device (module docs). Receiving chains rebuild as - // co-members redistribute over the re-handshaken sessions. - let chain = GroupSenderChain::generate(rng); - let (key_id, chain_key, iteration) = chain.snapshot(); - let pending = group - .members - .iter() - .filter(|m| m.peer != me) - .map(|m| PendingAnnounce { - peer: m.peer, - key_id, - chain_key: *chain_key, - iteration, - wire_id: None, - last_sent: 0, - }) - .collect(); - store.put_group( - &GroupRecord { - id: group.id, - name: group.name, - creator: group.creator, - members: group.members, - secret: group.secret, - prev_secret: None, - generation: group.generation, - sender_chain: postcard::to_allocvec(&chain) - .map_err(|_| StoreError::Serialization)?, - sent_since_rotation: 0, - pending, - }, - rng, - )?; - } - for message in &payload.group_messages { - store.put_group_message(message, rng)?; - } - for authority in &payload.group_authorities { - store.put_group_authority(authority, rng)?; - } - for record in &payload.local_metadata { - store.put_local_metadata(record, rng)?; + let record_count = validate_backup_payload(&payload)?; + let lock = acquire_store_lock(path)?; + if path.exists() { + return Err(StoreError::NotAStore); } - for message in &payload.note_messages { - store.put_note_message(message, rng)?; + let temporary = restore_temporary_path(path)?; + if temporary.exists() { + cleanup_restore_temporary(&temporary)?; } - for record in &payload.ephemeral { - store.put_ephemeral_record(record, rng)?; - } - for event in &payload.device_sync_events { - let decoded = DeviceSyncEvent::decode(event)?; - if let Some(manifest) = &payload.device_manifest { - decoded.verify(manifest)?; - } else { - return Err(StoreError::NotABackup); + ensure_restore_workspace(path, plain.len(), record_count)?; + let restored = populate_restored_store(&temporary, passphrase, profile, payload, rng); + let store = match restored { + Ok(store) => store, + Err(error) => { + cleanup_restore_temporary(&temporary)?; + return Err(error); } - store.put_device_sync_event(event, rng)?; + }; + store.validate_open_state()?; + migration::sync_database_for_replacement(&store.conn)?; + drop(store); + migration::sync_file(&temporary)?; + store_v2::sync_directory(migration::parent_directory(path))?; + restore_failpoint(1)?; + if path.exists() { + return Err(StoreError::NotAStore); } - restore_device_state( - &store, - &identity, - payload.device_manifest, - payload.local_device, - &payload.device_sync_events, - payload.created_at, - rng, - )?; + migration::atomic_replace(&temporary, path)?; + restore_failpoint(2)?; + store_v2::sync_directory(migration::parent_directory(path))?; + restore_failpoint(3)?; + let conn = Connection::open(path)?; + let database_lock = acquire_database_identity_lock(path)?; + let store = Store::open_v2_with_parts(path, passphrase, conn, database_lock, lock, false)?; + migration::cleanup_obsolete_siblings(&temporary)?; Ok(store) } /// The Argon2id profile this store was created with. fn kdf_profile(&self) -> Result { - let blob: Vec = self - .conn - .query_row("SELECT v FROM meta WHERE k = 'kdf'", [], |r| r.get(0)) - .map_err(|_| StoreError::NotAStore)?; - let (m, t, p): (u32, u32, u32) = - postcard::from_bytes(&blob).map_err(|_| StoreError::NotAStore)?; - Ok(KdfProfile { - m_cost_kib: m, - t_cost: t, - p_cost: p, - }) + Ok(self.kdf_profile) } /// Every stored message, in insertion order. pub(crate) fn all_messages(&self) -> Result> { - let mut stmt = self - .conn - .prepare("SELECT blob FROM messages ORDER BY rowid_")?; - let rows = stmt.query_map([], |r| r.get::<_, Vec>(0))?; let mut out = Vec::new(); - for row in rows { - let plain = self.k_messages.open(b"message", &row?)?; - out.push(postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?); + for row in self.rows::()? { + out.push(self.decode_message_row_ref(&row, None)?); } Ok(out) } /// Peers with a persisted ratchet session. fn session_peers(&self) -> Result> { - let mut stmt = self.conn.prepare("SELECT peer FROM sessions")?; - let rows = stmt.query_map([], |r| r.get::<_, Vec>(0))?; let mut out = Vec::new(); - for row in rows { - if let Ok(peer) = <[u8; 32]>::try_from(row?) { - out.push(peer); - } + for row in self.rows::()? { + let peer = *store_v2::AccountKey::decode(&row.logical_key)?.value(); + let _: kult_crypto::Session = decode_store_exact(&row.payload)?; + out.push(peer); } Ok(out) } @@ -630,36 +567,332 @@ impl Store { /// Record that any pre-restore session with this peer is dead and must /// be re-established (docs/07-storage.md §4). pub fn put_reset_marker(&self, peer: &[u8; 32]) -> Result<()> { - self.conn.execute( - "INSERT OR IGNORE INTO resets (peer) VALUES (?1)", - params![peer.as_slice()], + let mut rng = rand_core::OsRng; + self.put_equality::( + &store_v2::AccountKey::new(*peer), + peer, + store_v2::IndexKeys::none(), + &mut rng, )?; Ok(()) } /// All pending session-reset markers. pub fn reset_markers(&self) -> Result> { - let mut stmt = self.conn.prepare("SELECT peer FROM resets")?; - let rows = stmt.query_map([], |r| r.get::<_, Vec>(0))?; let mut out = Vec::new(); - for row in rows { - if let Ok(peer) = <[u8; 32]>::try_from(row?) { - out.push(peer); + for row in self.rows::()? { + let peer = *store_v2::AccountKey::decode(&row.logical_key)?.value(); + if row.payload.as_slice() != peer { + return Err(StoreError::LogicalKeyMismatch); } + out.push(peer); } Ok(out) } /// Remove a session-reset marker (the re-handshake was queued). pub fn clear_reset_marker(&self, peer: &[u8; 32]) -> Result<()> { - self.conn.execute( - "DELETE FROM resets WHERE peer = ?1", - params![peer.as_slice()], - )?; + self.delete_equality::(&store_v2::AccountKey::new(*peer))?; Ok(()) } } +fn populate_restored_store( + path: &Path, + passphrase: &[u8], + profile: KdfProfile, + mut payload: BackupPayload, + rng: &mut impl CryptoRngCore, +) -> Result { + let identity_bytes: Zeroizing<[u8; 64]> = Zeroizing::new( + payload.identity[..] + .try_into() + .map_err(|_| StoreError::NotABackup)?, + ); + payload.identity.zeroize(); + let identity = Identity::from_bytes(&identity_bytes); + let me = identity.public().ed; + let store = Store::create(path, passphrase, profile, rng)?; + store.put_identity(&identity, rng)?; + for contact in &payload.contacts { + store.put_contact(contact, rng)?; + } + for endpoint in &payload.contact_devices { + store.put_contact_device(endpoint, rng)?; + } + for message in &payload.messages { + store.put_message(message, rng)?; + } + for peer in &payload.reset_peers { + store.put_reset_marker(peer)?; + } + for group in payload.groups { + let chain = GroupSenderChain::generate(rng); + let (key_id, chain_key, iteration) = chain.snapshot(); + let pending = group + .members + .iter() + .filter(|member| member.peer != me) + .map(|member| PendingAnnounce { + peer: member.peer, + key_id, + chain_key: *chain_key, + iteration, + wire_id: None, + last_sent: 0, + }) + .collect(); + store.put_group( + &GroupRecord { + id: group.id, + name: group.name, + creator: group.creator, + members: group.members, + secret: group.secret, + prev_secret: None, + generation: group.generation, + sender_chain: postcard::to_allocvec(&chain) + .map_err(|_| StoreError::Serialization)?, + sent_since_rotation: 0, + pending, + }, + rng, + )?; + } + for message in &payload.group_messages { + store.put_group_message(message, rng)?; + } + for authority in &payload.group_authorities { + store.put_group_authority(authority, rng)?; + } + for record in &payload.local_metadata { + store.put_local_metadata(record, rng)?; + } + for message in &payload.note_messages { + store.put_note_message(message, rng)?; + } + for record in &payload.ephemeral { + store.put_ephemeral_record(record, rng)?; + } + for event in &payload.device_sync_events { + let decoded = DeviceSyncEvent::decode(event)?; + if let Some(manifest) = &payload.device_manifest { + decoded.verify(manifest)?; + } else { + return Err(StoreError::NotABackup); + } + store.put_device_sync_event(event, rng)?; + } + restore_device_state( + &store, + &identity, + payload.device_manifest, + payload.local_device, + &payload.device_sync_events, + payload.created_at, + rng, + )?; + Ok(store) +} + +fn validate_backup_payload(payload: &BackupPayload) -> Result { + if payload.identity.len() != 64 + || payload.contacts.len() > 100_000 + || payload.messages.len() > 10_000_000 + || payload.reset_peers.len() > 100_000 + || payload.groups.len() > 100_000 + || payload.group_messages.len() > 10_000_000 + || payload.group_authorities.len() > 100_000 + || payload.local_metadata.len() > 100_000 + || payload.note_messages.len() > 10_000_000 + || payload.ephemeral.len() > 10_000_000 + || payload.device_sync_events.len() > crate::MAX_DEVICE_SYNC_EVENTS + || payload.contact_devices.len() > 1_000_000 + { + return Err(StoreError::RecordBounds); + } + let total = [ + payload.contacts.len(), + payload.messages.len(), + payload.reset_peers.len(), + payload.groups.len(), + payload.group_messages.len(), + payload.group_authorities.len(), + payload.local_metadata.len(), + payload.note_messages.len(), + payload.ephemeral.len(), + payload.device_sync_events.len(), + payload.contact_devices.len(), + ] + .into_iter() + .try_fold(0u64, |total, count| { + total + .checked_add(u64::try_from(count).map_err(|_| StoreError::RecordBounds)?) + .ok_or(StoreError::RecordBounds) + })?; + if total > MAX_BACKUP_RECORDS { + return Err(StoreError::RecordBounds); + } + + let mut contacts = BTreeSet::new(); + for record in &payload.contacts { + if !contacts.insert(record.peer) { + return Err(StoreError::MigrationValidation); + } + validate_backup_record(record)?; + } + let mut message_ids = BTreeSet::new(); + for record in &payload.messages { + if !message_ids.insert(record.id) { + return Err(StoreError::MigrationValidation); + } + validate_backup_record(record)?; + } + if payload + .reset_peers + .iter() + .copied() + .collect::>() + .len() + != payload.reset_peers.len() + { + return Err(StoreError::MigrationValidation); + } + let mut groups = BTreeSet::new(); + for record in &payload.groups { + if !groups.insert(record.id) { + return Err(StoreError::MigrationValidation); + } + validate_backup_record(record)?; + } + let mut group_message_ids = BTreeSet::new(); + for record in &payload.group_messages { + if record.wire_body.is_some() || !group_message_ids.insert(record.id) { + return Err(StoreError::MigrationValidation); + } + validate_backup_record(record)?; + } + let mut authorities = BTreeSet::new(); + for record in &payload.group_authorities { + if !authorities.insert(record.group) { + return Err(StoreError::MigrationValidation); + } + validate_backup_record(record)?; + } + let mut metadata_keys = BTreeSet::new(); + for record in &payload.local_metadata { + let key = postcard::to_allocvec(&record.key()).map_err(|_| StoreError::Serialization)?; + if !metadata_keys.insert(key) { + return Err(StoreError::MigrationValidation); + } + validate_backup_record(record)?; + } + let mut note_ids = BTreeSet::new(); + for record in &payload.note_messages { + if !note_ids.insert(record.id) { + return Err(StoreError::MigrationValidation); + } + validate_backup_record(record)?; + } + let mut ephemeral_keys = BTreeSet::new(); + for record in &payload.ephemeral { + if record.state == EphemeralState::Active || !record.transfer_ids.is_empty() { + return Err(StoreError::MigrationValidation); + } + let key = postcard::to_allocvec(&(record.conversation, record.author, record.content_id)) + .map_err(|_| StoreError::Serialization)?; + if !ephemeral_keys.insert(key) { + return Err(StoreError::MigrationValidation); + } + validate_backup_record(record)?; + } + let mut devices = BTreeSet::new(); + for record in &payload.contact_devices { + if !contacts.contains(&record.account) || !devices.insert((record.account, record.device)) { + return Err(StoreError::MigrationValidation); + } + validate_backup_record(record)?; + } + for event in &payload.device_sync_events { + if event.is_empty() || event.len() > crate::MAX_DEVICE_SYNC_EVENT_BYTES { + return Err(StoreError::RecordBounds); + } + } + if payload.device_manifest.is_some() != payload.local_device.is_some() + || (payload.device_manifest.is_none() && !payload.device_sync_events.is_empty()) + { + return Err(StoreError::MigrationValidation); + } + Ok(total) +} + +fn validate_backup_record(record: &T) -> Result<()> { + let encoded = postcard::to_allocvec(record).map_err(|_| StoreError::Serialization)?; + if encoded.len() > store_v2::MAX_RECORD_BYTES { + return Err(StoreError::RecordBounds); + } + Ok(()) +} + +fn restore_temporary_path(path: &Path) -> Result { + let name = path.file_name().ok_or(StoreError::NotAStore)?; + let mut temporary = name.to_os_string(); + temporary.push(".restore-v2-sibling"); + Ok(path.with_file_name(temporary)) +} + +fn ensure_restore_workspace(path: &Path, logical_bytes: usize, record_count: u64) -> Result<()> { + let logical_bytes = + u64::try_from(logical_bytes).map_err(|_| StoreError::InsufficientMigrationSpace)?; + let required = logical_bytes + .checked_mul(2) + .and_then(|bytes| { + record_count + .checked_mul(ESTIMATED_RESTORED_ROW_BYTES) + .and_then(|overhead| bytes.checked_add(overhead)) + }) + .and_then(|bytes| bytes.checked_add(RESTORE_SPACE_RESERVE)) + .ok_or(StoreError::InsufficientMigrationSpace)?; + if fs2::available_space(migration::parent_directory(path))? < required { + return Err(StoreError::InsufficientMigrationSpace); + } + Ok(()) +} + +fn cleanup_restore_temporary(path: &Path) -> Result<()> { + if path.exists() { + fs::remove_file(path)?; + } + migration::cleanup_obsolete_siblings(path)?; + store_v2::sync_directory(migration::parent_directory(path)) +} + +pub(crate) fn cleanup_completed_restore(path: &Path) -> Result<()> { + let temporary = restore_temporary_path(path)?; + if temporary.exists() { + return Err(StoreError::ReplacementRecovery); + } + migration::cleanup_obsolete_siblings(&temporary) +} + +#[cfg(test)] +static RESTORE_FAILPOINT: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +#[cfg(test)] +fn set_restore_failpoint(phase: u8) { + RESTORE_FAILPOINT.store(phase, std::sync::atomic::Ordering::SeqCst); +} + +fn restore_failpoint(phase: u8) -> Result<()> { + #[cfg(test)] + if RESTORE_FAILPOINT.load(std::sync::atomic::Ordering::SeqCst) == phase { + RESTORE_FAILPOINT.store(0, std::sync::atomic::Ordering::SeqCst); + return Err(StoreError::MigrationValidation); + } + let _ = phase; + Ok(()) +} + #[allow(clippy::too_many_arguments)] fn restore_device_state( store: &Store, @@ -747,3 +980,113 @@ fn restore_device_state( )?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::store_v2::TableSpec; + use rand::{rngs::StdRng, SeedableRng}; + use rusqlite::params; + + const TEST_KDF: KdfProfile = KdfProfile { + m_cost_kib: 8, + t_cost: 1, + p_cost: 1, + }; + + #[test] + fn restore_restarts_safely_at_every_replacement_phase() { + for phase in 1..=3 { + let directory = tempfile::tempdir().unwrap(); + let source_path = directory.path().join(format!("source-{phase}.db")); + let target_path = directory.path().join(format!("target-{phase}.db")); + let mut rng = StdRng::seed_from_u64(0x7000 + phase as u64); + let source = Store::create(&source_path, b"source-pass", TEST_KDF, &mut rng).unwrap(); + let identity = Identity::generate(&mut rng); + let expected = identity.public(); + source.put_identity(&identity, &mut rng).unwrap(); + let (backup, mnemonic) = source.export_backup(123, &mut rng).unwrap(); + drop(source); + + set_restore_failpoint(phase); + assert!(matches!( + Store::restore_backup( + &target_path, + &backup, + &mnemonic, + b"target-pass", + TEST_KDF, + &mut rng, + ), + Err(StoreError::MigrationValidation) + )); + let restored = if phase == 1 { + Store::restore_backup( + &target_path, + &backup, + &mnemonic, + b"target-pass", + TEST_KDF, + &mut rng, + ) + .unwrap() + } else { + Store::open(&target_path, b"target-pass").unwrap() + }; + assert_eq!(restored.get_identity().unwrap().unwrap().public(), expected); + assert!(!restore_temporary_path(&target_path).unwrap().exists()); + } + } + + #[test] + fn backup_plaintext_contains_logical_records_not_store_artifacts() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("logical-backup.db"); + let mut rng = StdRng::seed_from_u64(0x7004); + let store = Store::create(&path, b"source-pass", TEST_KDF, &mut rng).unwrap(); + let identity = Identity::generate(&mut rng); + store.put_identity(&identity, &mut rng).unwrap(); + let message = MessageRecord { + id: [3; 16], + peer: [4; 32], + direction: crate::Direction::Inbound, + state: crate::DeliveryState::Received, + timestamp: 5, + body: b"logical backup record".to_vec(), + wire_id: None, + }; + store.put_message(&message, &mut rng).unwrap(); + let (locator, opaque_index, wrapped_row): (Vec, Vec, Vec) = store + .conn + .query_row( + "SELECT locator, unique_index, blob FROM store_records + WHERE table_domain = ?1", + params![store_v2::MessageRows::DOMAIN], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + let database_id = store.metadata.database_id; + + let (backup, mnemonic) = store.export_backup(6, &mut rng).unwrap(); + let entropy = mnemonic_to_entropy(&mnemonic).unwrap(); + let salt: [u8; 16] = backup[16..32].try_into().unwrap(); + let kek = derive_kek(&entropy[..], &salt, TEST_KDF).unwrap(); + let plain = StorageKey::from_bytes(*kek) + .open(BACKUP_AD, &backup[HEADER_LEN..]) + .unwrap(); + let decoded: BackupPayload = decode_exact(&plain).unwrap(); + assert_eq!(decoded.messages, vec![message]); + assert_eq!(decoded.identity, identity.to_bytes().to_vec()); + assert!(!plain.starts_with(b"SQLite format 3")); + for artifact in [ + database_id.as_slice(), + locator.as_slice(), + opaque_index.as_slice(), + wrapped_row.as_slice(), + ] { + assert!(!plain + .windows(artifact.len()) + .any(|candidate| candidate == artifact)); + } + } +} diff --git a/crates/kult-store/src/devices.rs b/crates/kult-store/src/devices.rs index 4923c45..223ca08 100644 --- a/crates/kult-store/src/devices.rs +++ b/crates/kult-store/src/devices.rs @@ -7,20 +7,19 @@ use std::collections::HashSet; use rand_core::CryptoRngCore; -use rusqlite::{params, OptionalExtension}; use serde::{Deserialize, Serialize}; use zeroize::Zeroizing; use kult_crypto::{DeviceCertificate, DeviceManifest, GroupSenderChain, Identity}; use crate::{ - ContactRecord, DeliveryState, EphemeralRecord, EphemeralState, GroupAuthorityRecord, - GroupMember, GroupMessageRecord, GroupRecord, LocalMetadataRecord, MessageRecord, - NoteMessageRecord, PendingAnnounce, Result, Store, StoreError, THEME_PREFERENCE_KEY, + decode_exact, store_v2, ContactRecord, DeliveryState, EphemeralRecord, EphemeralState, + GroupAuthorityRecord, GroupMember, GroupMessageRecord, GroupRecord, LocalMetadataRecord, + MessageRecord, NoteMessageRecord, PendingAnnounce, Result, Store, StoreError, + THEME_PREFERENCE_KEY, }; -const DEVICE_STATE_AD: &[u8] = b"device-state-v1"; -const DEVICE_SYNC_AD: &[u8] = b"device-sync-v1"; +const DEVICE_SYNC_DIGEST_DOMAIN: &[u8] = b"Komms-Store-Device-Sync-Digest-v2"; /// Maximum authenticated sync-event bytes stored in one row. pub const MAX_DEVICE_SYNC_EVENT_BYTES: usize = 1024 * 1024; /// Maximum durable sync events before compaction must make progress. @@ -386,10 +385,11 @@ impl Store { state.validate(&account)?; let plain = Zeroizing::new(postcard::to_allocvec(state).map_err(|_| StoreError::Serialization)?); - let sealed = self.k_devices.seal(DEVICE_STATE_AD, &plain, rng); - self.conn.execute( - "INSERT OR REPLACE INTO device_state (id, blob) VALUES (1, ?1)", - params![sealed], + self.put_equality::( + &store_v2::SingletonKey, + &plain, + store_v2::IndexKeys::none(), + rng, )?; Ok(()) } @@ -408,84 +408,57 @@ impl Store { { return Err(StoreError::Serialization); } - let encoded = postcard::to_allocvec(endpoint).map_err(|_| StoreError::Serialization)?; - let sealed = self.k_devices.seal(b"contact-device-v1", &encoded, rng); - let mut statement = self - .conn - .prepare("SELECT rowid_, blob FROM contact_devices ORDER BY rowid_")?; - let rows = statement.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - })?; - for row in rows { - let (rowid, stored) = row?; - let plain = self.k_devices.open(b"contact-device-v1", &stored)?; - let decoded: ContactDeviceRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if decoded.account == endpoint.account && decoded.device == endpoint.device { - self.conn.execute( - "UPDATE contact_devices SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; - return Ok(()); - } - } - drop(statement); - self.conn.execute( - "INSERT INTO contact_devices (blob) VALUES (?1)", - params![sealed], + let encoded = + Zeroizing::new(postcard::to_allocvec(endpoint).map_err(|_| StoreError::Serialization)?); + self.put_equality::( + &store_v2::AccountDeviceKey::new(endpoint.account, endpoint.device), + &encoded, + store_v2::IndexKeys::contact_device(&store_v2::AccountKey::new(endpoint.account)), + rng, )?; Ok(()) } /// Every sealed contact-device endpoint in insertion order. pub fn contact_devices(&self) -> Result> { - let mut statement = self - .conn - .prepare("SELECT blob FROM contact_devices ORDER BY rowid_")?; - let rows = statement.query_map([], |row| row.get::<_, Vec>(0))?; let mut endpoints = Vec::new(); - for row in rows { - let plain = self.k_devices.open(b"contact-device-v1", &row?)?; - endpoints.push(postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?); + for row in self.rows::()? { + let endpoint: ContactDeviceRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::AccountDeviceKey::new( + endpoint.account, + endpoint.device, + ))?; + endpoints.push(endpoint); } Ok(endpoints) } /// Active endpoints known for one stable contact account. pub fn contact_devices_for(&self, account: &[u8; 32]) -> Result> { - Ok(self - .contact_devices()? - .into_iter() - .filter(|endpoint| &endpoint.account == account && endpoint.revoked_at.is_none()) - .collect()) + let mut endpoints = Vec::new(); + for row in self.rows_by_index::( + &store_v2::AccountKey::new(*account), + )? { + let endpoint: ContactDeviceRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::AccountDeviceKey::new( + endpoint.account, + endpoint.device, + ))?; + if endpoint.account != *account { + return Err(StoreError::LogicalKeyMismatch); + } + if endpoint.revoked_at.is_none() { + endpoints.push(endpoint); + } + } + Ok(endpoints) } /// Delete one exact sealed contact-device endpoint. pub fn delete_contact_device(&self, account: &[u8; 32], device: &[u8; 32]) -> Result<()> { - let mut statement = self - .conn - .prepare("SELECT rowid_, blob FROM contact_devices ORDER BY rowid_")?; - let rows = statement.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - })?; - let mut target = None; - for row in rows { - let (rowid, stored) = row?; - let plain = self.k_devices.open(b"contact-device-v1", &stored)?; - let decoded: ContactDeviceRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if &decoded.account == account && &decoded.device == device { - target = Some(rowid); - break; - } - } - drop(statement); - if let Some(rowid) = target { - self.conn.execute( - "DELETE FROM contact_devices WHERE rowid_ = ?1", - params![rowid], - )?; - } + self.delete_equality::(&store_v2::AccountDeviceKey::new( + *account, *device, + ))?; Ok(()) } @@ -498,37 +471,39 @@ impl Store { new_device: &[u8; 32], rng: &mut impl CryptoRngCore, ) -> Result<()> { - let mut statement = self - .conn - .prepare("SELECT rowid_, blob FROM message_device_delivery ORDER BY rowid_")?; - let rows = statement.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - })?; let mut replacements = Vec::new(); - for row in rows { - let (rowid, stored) = row?; - let plain = self - .k_devices - .open(b"message-device-delivery-v1", &stored)?; - let mut delivery: MessageDeviceDeliveryRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; + for row in self.rows_by_index::( + &store_v2::AccountKey::new(*account), + )? { + let mut delivery: MessageDeviceDeliveryRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::MessageDeviceKey::new( + delivery.message, + delivery.device, + ))?; if &delivery.account == account && &delivery.device == old_device { delivery.device = *new_device; - let encoded = - postcard::to_allocvec(&delivery).map_err(|_| StoreError::Serialization)?; - let sealed = self - .k_devices - .seal(b"message-device-delivery-v1", &encoded, rng); - replacements.push((rowid, sealed)); + replacements.push((row.locator, delivery)); } } - drop(statement); - for (rowid, sealed) in replacements { - self.conn.execute( - "UPDATE message_device_delivery SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], + let tx = self.conn.unchecked_transaction()?; + for (locator, delivery) in replacements { + let new_key = store_v2::MessageDeviceKey::new(delivery.message, *new_device); + let encoded = Zeroizing::new( + postcard::to_allocvec(&delivery).map_err(|_| StoreError::Serialization)?, + ); + self.delete_row_on::(&tx, &locator)?; + self.put_equality_on::( + &tx, + &new_key, + &encoded, + store_v2::IndexKeys::message_device_delivery( + &store_v2::ContentKey::new(delivery.message), + &store_v2::AccountKey::new(delivery.account), + ), + rng, )?; } + tx.commit()?; Ok(()) } @@ -544,35 +519,16 @@ impl Store { { return Err(StoreError::Serialization); } - let encoded = postcard::to_allocvec(delivery).map_err(|_| StoreError::Serialization)?; - let sealed = self - .k_devices - .seal(b"message-device-delivery-v1", &encoded, rng); - let mut statement = self - .conn - .prepare("SELECT rowid_, blob FROM message_device_delivery ORDER BY rowid_")?; - let rows = statement.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - })?; - for row in rows { - let (rowid, stored) = row?; - let plain = self - .k_devices - .open(b"message-device-delivery-v1", &stored)?; - let decoded: MessageDeviceDeliveryRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if decoded.message == delivery.message && decoded.device == delivery.device { - self.conn.execute( - "UPDATE message_device_delivery SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; - return Ok(()); - } - } - drop(statement); - self.conn.execute( - "INSERT INTO message_device_delivery (blob) VALUES (?1)", - params![sealed], + let encoded = + Zeroizing::new(postcard::to_allocvec(delivery).map_err(|_| StoreError::Serialization)?); + self.put_equality::( + &store_v2::MessageDeviceKey::new(delivery.message, delivery.device), + &encoded, + store_v2::IndexKeys::message_device_delivery( + &store_v2::ContentKey::new(delivery.message), + &store_v2::AccountKey::new(delivery.account), + ), + rng, )?; Ok(()) } @@ -582,36 +538,31 @@ impl Store { &self, message: &[u8; 16], ) -> Result> { - let mut statement = self - .conn - .prepare("SELECT blob FROM message_device_delivery ORDER BY rowid_")?; - let rows = statement.query_map([], |row| row.get::<_, Vec>(0))?; let mut deliveries = Vec::new(); - for row in rows { - let plain = self.k_devices.open(b"message-device-delivery-v1", &row?)?; - let delivery: MessageDeviceDeliveryRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if &delivery.message == message { - deliveries.push(delivery); + for row in self.rows_by_index::( + &store_v2::ContentKey::new(*message), + )? { + let delivery: MessageDeviceDeliveryRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::MessageDeviceKey::new( + delivery.message, + delivery.device, + ))?; + if delivery.message != *message { + return Err(StoreError::LogicalKeyMismatch); } + deliveries.push(delivery); } Ok(deliveries) } /// Load and validate the complete local linked-device state, if enabled. pub fn get_device_state(&self) -> Result> { - let sealed: Option> = self - .conn - .query_row("SELECT blob FROM device_state WHERE id = 1", [], |row| { - row.get(0) - }) - .optional()?; - let Some(sealed) = sealed else { + let Some(row) = self.get_equality::(&store_v2::SingletonKey)? + else { return Ok(None); }; - let plain = Zeroizing::new(self.k_devices.open(DEVICE_STATE_AD, &sealed)?); - let state: DeviceStateRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; + row.verify_key(&store_v2::SingletonKey)?; + let state: DeviceStateRecord = decode_exact(&row.payload)?; let account = self.get_identity()?.ok_or(StoreError::NotAStore)?; state.validate(&account)?; Ok(Some(state)) @@ -627,34 +578,34 @@ impl Store { if event.is_empty() || event.len() > MAX_DEVICE_SYNC_EVENT_BYTES { return Err(StoreError::Serialization); } - let existing = self.device_sync_events()?; - if existing.iter().any(|stored| stored == event) { + let digest = self.device_sync_digest(event); + if self + .row_by_unique::(&digest)? + .is_some() + { return Ok(false); } - if existing.len() >= MAX_DEVICE_SYNC_EVENTS { + if self.count_rows::()? >= MAX_DEVICE_SYNC_EVENTS as u64 { return Err(StoreError::Serialization); } - let sealed = self.k_devices.seal(DEVICE_SYNC_AD, event, rng); - self.conn.execute( - "INSERT INTO device_sync (blob) VALUES (?1)", - params![sealed], + self.append::( + &digest, + event, + store_v2::IndexKeys::device_sync(&digest), + rng, )?; Ok(true) } /// Return every opaque authenticated sync event in insertion order. pub fn device_sync_events(&self) -> Result>> { - let mut statement = self - .conn - .prepare("SELECT blob FROM device_sync ORDER BY rowid_")?; - let rows = statement.query_map([], |row| row.get::<_, Vec>(0))?; let mut events = Vec::new(); - for row in rows { - let event = self.k_devices.open(DEVICE_SYNC_AD, &row?)?; - if event.is_empty() || event.len() > MAX_DEVICE_SYNC_EVENT_BYTES { + for row in self.rows::()? { + if row.payload.is_empty() || row.payload.len() > MAX_DEVICE_SYNC_EVENT_BYTES { return Err(StoreError::Serialization); } - events.push(event); + row.verify_key(&self.device_sync_digest(&row.payload))?; + events.push(row.payload.to_vec()); } Ok(events) } @@ -663,25 +614,94 @@ impl Store { /// compaction snapshot commits. Event bytes remain sealed lookup keys. pub fn retain_device_sync_events(&self, retain: &[Vec]) -> Result<()> { let wanted: HashSet<&[u8]> = retain.iter().map(Vec::as_slice).collect(); - let mut statement = self - .conn - .prepare("SELECT rowid_, blob FROM device_sync ORDER BY rowid_")?; - let rows = statement.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - })?; let mut remove = Vec::new(); - for row in rows { - let (rowid, sealed) = row?; - let event = self.k_devices.open(DEVICE_SYNC_AD, &sealed)?; - if !wanted.contains(event.as_slice()) { - remove.push(rowid); + for row in self.rows::()? { + row.verify_key(&self.device_sync_digest(&row.payload))?; + if !wanted.contains(row.payload.as_slice()) { + remove.push(row.rowid); } } - drop(statement); + let tx = self.conn.unchecked_transaction()?; for rowid in remove { - self.conn - .execute("DELETE FROM device_sync WHERE rowid_ = ?1", params![rowid])?; + self.delete_rowid_on::(&tx, rowid)?; } + tx.commit()?; Ok(()) } + + fn device_sync_digest(&self, event: &[u8]) -> store_v2::DigestKey { + let mut input = Vec::with_capacity(DEVICE_SYNC_DIGEST_DOMAIN.len() + event.len()); + input.extend_from_slice(DEVICE_SYNC_DIGEST_DOMAIN); + input.extend_from_slice(event); + store_v2::DigestKey::new(self.index_root.hmac_sha256(&input)) + } + + pub(crate) fn validate_device_logical_rows(&self) -> Result<()> { + let account = self.get_identity()?; + self.validate_rows::(|row| { + row.verify_key(&store_v2::SingletonKey)?; + row.verify_indexes(&store_v2::IndexKeys::none())?; + let state: DeviceStateRecord = decode_exact(&row.payload)?; + state.validate(account.as_ref().ok_or(StoreError::NotAStore)?) + })?; + self.validate_rows::(|row| { + if row.payload.is_empty() || row.payload.len() > MAX_DEVICE_SYNC_EVENT_BYTES { + return Err(StoreError::Serialization); + } + let digest = self.device_sync_digest(&row.payload); + let stored = store_v2::DigestKey::decode(&row.logical_key)?; + if stored.value() != digest.value() { + return Err(StoreError::LogicalKeyMismatch); + } + row.verify_indexes(&store_v2::IndexKeys::device_sync(&digest)) + })?; + if self.count_rows::()? > MAX_DEVICE_SYNC_EVENTS as u64 { + return Err(StoreError::Serialization); + } + self.validate_rows::(|row| { + let endpoint: ContactDeviceRecord = decode_exact(&row.payload)?; + validate_contact_device(&endpoint)?; + let key = store_v2::AccountDeviceKey::decode(&row.logical_key)?; + if key.account() != &endpoint.account || key.device() != &endpoint.device { + return Err(StoreError::LogicalKeyMismatch); + } + row.verify_indexes(&store_v2::IndexKeys::contact_device( + &store_v2::AccountKey::new(endpoint.account), + )) + })?; + self.validate_rows::(|row| { + let delivery: MessageDeviceDeliveryRecord = decode_exact(&row.payload)?; + validate_message_device_delivery(&delivery)?; + let key = store_v2::MessageDeviceKey::decode(&row.logical_key)?; + if key.message() != &delivery.message || key.device() != &delivery.device { + return Err(StoreError::LogicalKeyMismatch); + } + row.verify_indexes(&store_v2::IndexKeys::message_device_delivery( + &store_v2::ContentKey::new(delivery.message), + &store_v2::AccountKey::new(delivery.account), + )) + }) + } +} + +fn validate_contact_device(endpoint: &ContactDeviceRecord) -> Result<()> { + if endpoint.account == [0u8; 32] + || endpoint.device == [0u8; 32] + || (endpoint.certificate.is_empty() && endpoint.account != endpoint.device) + || (endpoint.manifest_generation == 0) != (endpoint.manifest_state_id == [0u8; 32]) + || endpoint.revoked_at.is_some() != endpoint.revoked_after_counter.is_some() + { + return Err(StoreError::Serialization); + } + Ok(()) +} + +fn validate_message_device_delivery(delivery: &MessageDeviceDeliveryRecord) -> Result<()> { + if delivery.message == [0u8; 16] + || delivery.account == [0u8; 32] + || delivery.device == [0u8; 32] + { + return Err(StoreError::Serialization); + } + Ok(()) } diff --git a/crates/kult-store/src/ephemeral.rs b/crates/kult-store/src/ephemeral.rs index c0a7044..dbc5e2f 100644 --- a/crates/kult-store/src/ephemeral.rs +++ b/crates/kult-store/src/ephemeral.rs @@ -1,12 +1,10 @@ //! Sealed local expiry markers and durable tombstones (ADR-0021). use rand_core::CryptoRngCore; -use rusqlite::params; use serde::{Deserialize, Serialize}; +use zeroize::Zeroizing; -use crate::{Result, Store, StoreError}; - -const EPHEMERAL_AD: &[u8] = b"ephemeral-v1"; +use crate::{decode_exact, store_v2, Result, Store, StoreError}; /// Conversation scope for one ephemeral content id. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -69,19 +67,14 @@ impl Store { { return Err(StoreError::Serialization); } - let plain = postcard::to_allocvec(record).map_err(|_| StoreError::Serialization)?; - let sealed = self.k_ephemeral.seal(EPHEMERAL_AD, &plain, rng); - if let Some(rowid) = - self.ephemeral_rowid(&record.conversation, &record.author, &record.content_id)? - { - self.conn.execute( - "UPDATE ephemeral SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; - } else { - self.conn - .execute("INSERT INTO ephemeral (blob) VALUES (?1)", params![sealed])?; - } + let plain = + Zeroizing::new(postcard::to_allocvec(record).map_err(|_| StoreError::Serialization)?); + self.put_equality::( + &ephemeral_key(&record.conversation, &record.author, &record.content_id)?, + &plain, + store_v2::IndexKeys::none(), + rng, + )?; Ok(()) } @@ -92,51 +85,63 @@ impl Store { author: &[u8; 32], content_id: &[u8; 16], ) -> Result> { - Ok(self.ephemeral_records()?.into_iter().find(|record| { - &record.conversation == conversation - && &record.author == author - && &record.content_id == content_id - })) + let key = ephemeral_key(conversation, author, content_id)?; + let Some(row) = self.get_equality::(&key)? else { + return Ok(None); + }; + row.verify_key(&key)?; + let record: EphemeralRecord = decode_exact(&row.payload)?; + if record.conversation != *conversation + || record.author != *author + || record.content_id != *content_id + { + return Err(StoreError::LogicalKeyMismatch); + } + Ok(Some(record)) } /// Every sealed marker, including durable consumed/expired tombstones. pub fn ephemeral_records(&self) -> Result> { - let mut stmt = self - .conn - .prepare("SELECT blob FROM ephemeral ORDER BY rowid_")?; - let rows = stmt.query_map([], |row| row.get::<_, Vec>(0))?; let mut out = Vec::new(); - for row in rows { - let plain = self.k_ephemeral.open(EPHEMERAL_AD, &row?)?; - out.push(postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?); + for row in self.rows::()? { + let record: EphemeralRecord = decode_exact(&row.payload)?; + row.verify_key(&ephemeral_key( + &record.conversation, + &record.author, + &record.content_id, + )?)?; + out.push(record); } Ok(out) } - fn ephemeral_rowid( - &self, - conversation: &EphemeralConversation, - author: &[u8; 32], - content_id: &[u8; 16], - ) -> Result> { - let mut stmt = self - .conn - .prepare("SELECT rowid_, blob FROM ephemeral ORDER BY rowid_")?; - let rows = stmt.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - })?; - for row in rows { - let (rowid, sealed) = row?; - let plain = self.k_ephemeral.open(EPHEMERAL_AD, &sealed)?; - let record: EphemeralRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if &record.conversation == conversation - && &record.author == author - && &record.content_id == content_id + pub(crate) fn validate_ephemeral_logical_rows(&self) -> Result<()> { + self.validate_rows::(|row| { + let record: EphemeralRecord = decode_exact(&row.payload)?; + if record.expires_at == 0 + || (record.mode == EphemeralMode::DisappearingText + && !record.transfer_ids.is_empty()) { - return Ok(Some(rowid)); + return Err(StoreError::Serialization); } - } - Ok(None) + row.verify_key(&ephemeral_key( + &record.conversation, + &record.author, + &record.content_id, + )?)?; + row.verify_indexes(&store_v2::IndexKeys::none()) + }) } } + +fn ephemeral_key( + conversation: &EphemeralConversation, + author: &[u8; 32], + content_id: &[u8; 16], +) -> Result { + let (kind, conversation) = match conversation { + EphemeralConversation::Pairwise(id) => (0, *id), + EphemeralConversation::Group(id) => (1, *id), + }; + store_v2::EphemeralKey::new(kind, conversation, *author, *content_id) +} diff --git a/crates/kult-store/src/lib.rs b/crates/kult-store/src/lib.rs index 0ada5a3..bd02800 100644 --- a/crates/kult-store/src/lib.rs +++ b/crates/kult-store/src/lib.rs @@ -16,7 +16,9 @@ use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; use rand_core::CryptoRngCore; -use rusqlite::{params, Connection, OptionalExtension, Transaction, TransactionBehavior}; +#[cfg(test)] +use rusqlite::params; +use rusqlite::{Connection, Transaction, TransactionBehavior}; use serde::{Deserialize, Serialize}; use zeroize::Zeroizing; @@ -27,11 +29,16 @@ mod backup; mod devices; mod ephemeral; mod local_metadata; +mod maintenance; mod media; +mod migration; mod note; +#[cfg(test)] +mod opaque_tests; +#[cfg(test)] +mod scale_bench; mod scheduled; -#[doc(hidden)] -pub mod store_v2; +mod store_v2; pub use backup::BACKUP_MAGIC; pub use devices::{ @@ -54,6 +61,10 @@ pub use local_metadata::{ MAX_LOCAL_METADATA_STRING_BYTES, MAX_PINS, MAX_UI_PREFERENCE_VALUE_BYTES, THEME_PREFERENCES, THEME_PREFERENCE_KEY, THEME_SEMANTIC_ROLES, }; +pub use maintenance::{ + StorageMaintenanceOptions, StorageMaintenanceReport, MAX_MAINTENANCE_VACUUM_PAGES, + MAX_MAINTENANCE_WAL_BYTES, +}; pub use media::{ MediaDirection, MediaLimits, MediaObjectRecord, MediaReconciliation, MediaRecord, MediaScope, MediaTransferRecord, MediaTransferState, MediaUsage, DEFAULT_MEDIA_STORE_QUOTA, @@ -159,6 +170,21 @@ pub enum StoreError { UnsupportedRecordVersion, /// A versioned logical record exceeds the internal migration bound. RecordBounds, + /// An opaque history cursor is malformed, forged, stale, or belongs to + /// another database or conversation. + InvalidCursor, + /// The legacy database does not match a released schema fixture. + UnsupportedLegacySchema, + /// A migration or restore does not have enough same-filesystem workspace. + InsufficientMigrationSpace, + /// A sibling migration checkpoint no longer matches its source database. + MigrationSourceChanged, + /// Atomic replacement recovery found an ambiguous set of sibling files. + ReplacementRecovery, + /// A bounded migration count or referential check failed. + MigrationValidation, + /// A requested maintenance bound exceeds the supported per-call limit. + MaintenanceBounds, } impl std::fmt::Display for StoreError { @@ -217,6 +243,21 @@ impl std::fmt::Display for StoreError { f.write_str("sealed logical record version is unsupported") } Self::RecordBounds => f.write_str("sealed logical record exceeds its bound"), + Self::InvalidCursor => f.write_str("invalid or stale opaque history cursor"), + Self::UnsupportedLegacySchema => { + f.write_str("legacy store schema is not a released Komms schema") + } + Self::InsufficientMigrationSpace => { + f.write_str("insufficient temporary space for store replacement") + } + Self::MigrationSourceChanged => { + f.write_str("migration source changed after its checkpoint") + } + Self::ReplacementRecovery => { + f.write_str("store replacement requires explicit recovery") + } + Self::MigrationValidation => f.write_str("store migration validation failed"), + Self::MaintenanceBounds => f.write_str("store maintenance bound exceeded"), } } } @@ -247,6 +288,25 @@ impl From for StoreError { /// Convenience alias. pub type Result = std::result::Result; +fn decode_exact(bytes: &[u8]) -> Result +where + T: for<'de> Deserialize<'de>, +{ + let (value, remainder) = + postcard::take_from_bytes(bytes).map_err(|_| StoreError::Serialization)?; + if !remainder.is_empty() { + return Err(StoreError::Serialization); + } + Ok(value) +} + +fn direction_code(direction: Direction) -> u8 { + match direction { + Direction::Outbound => 0, + Direction::Inbound => 1, + } +} + /// Direction of a stored message. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum Direction { @@ -291,6 +351,43 @@ pub struct MessageRecord { pub wire_id: Option<[u8; 16]>, } +/// Stable opaque continuation token for one database and conversation. +/// +/// The bytes reveal neither a logical identifier nor a usable SQLite row +/// number and fail closed if replayed against another store or conversation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryCursor(Vec); + +impl HistoryCursor { + /// Reconstruct a cursor received through an RPC or FFI boundary. + pub fn from_bytes(bytes: Vec) -> Self { + Self(bytes) + } + + /// Opaque bytes suitable for persistence by a caller. + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +/// One bounded page of pairwise message history. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MessagePage { + /// Records in stable insertion order. + pub records: Vec, + /// Continuation after the final returned row, when more rows may exist. + pub next: Option, +} + +/// One bounded page of group message history. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GroupMessagePage { + /// Records in stable insertion order. + pub records: Vec, + /// Continuation after the final returned row, when more rows may exist. + pub next: Option, +} + /// Complete durable consequences of accepting one ordinary pairwise message. /// /// The session is a candidate advanced from the last durable session. Applying @@ -511,7 +608,8 @@ const WRAP_AD: &[u8] = b"KK-store-wrap-v1"; pub const MAX_PENDING_ENVELOPES: usize = 2_048; /// Maximum aggregate sealed bytes retained by the deferred inbox. pub const MAX_PENDING_BYTES: usize = 64 * 1024 * 1024; -const SCHEMA: &str = " +#[cfg(test)] +pub(crate) const LEGACY_SCHEMA_CURRENT: &str = " CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v BLOB NOT NULL); CREATE TABLE IF NOT EXISTS identity (id INTEGER PRIMARY KEY CHECK (id = 1), blob BLOB NOT NULL); CREATE TABLE IF NOT EXISTS sessions (peer BLOB PRIMARY KEY, blob BLOB NOT NULL); @@ -573,6 +671,12 @@ fn store_lock_path(path: &Path) -> Result { /// The sidecar intentionally remains after drop: unlinking a lock file can /// split contenders across different inodes. Dropping the returned handle /// releases the advisory lock. +fn is_lock_contention(error: &std::io::Error) -> bool { + let expected = fs2::lock_contended_error(); + error.kind() == std::io::ErrorKind::WouldBlock + || (expected.raw_os_error().is_some() && error.raw_os_error() == expected.raw_os_error()) +} + fn acquire_store_lock(path: &Path) -> Result { let lock_path = store_lock_path(path)?; let mut options = OpenOptions::new(); @@ -585,7 +689,7 @@ fn acquire_store_lock(path: &Path) -> Result { let lock = options.open(lock_path)?; match fs2::FileExt::try_lock_exclusive(&lock) { Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + Err(error) if is_lock_contention(&error) => { return Err(StoreError::AlreadyOpen); } Err(error) => return Err(StoreError::Io(error)), @@ -610,9 +714,7 @@ fn acquire_database_identity_lock(path: &Path) -> Result> { let database = OpenOptions::new().read(true).write(true).open(path)?; match fs2::FileExt::try_lock_exclusive(&database) { Ok(()) => Ok(Some(database)), - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { - Err(StoreError::AlreadyOpen) - } + Err(error) if is_lock_contention(&error) => Err(StoreError::AlreadyOpen), Err(error) => Err(StoreError::Io(error)), } } @@ -626,23 +728,14 @@ fn acquire_database_identity_lock(path: &Path) -> Result> { /// An open Komms store with its encrypted domains unlocked. pub struct Store { conn: Connection, - k_identity: StorageKey, - k_sessions: StorageKey, - k_capabilities: StorageKey, - k_messages: StorageKey, - k_queue: StorageKey, - k_contacts: StorageKey, - k_prekeys: StorageKey, - k_pending: StorageKey, - /// One key for the three group tables; the associated-data strings - /// (`group` / `group-chain` / `group-msg`) keep the domains disjoint. - k_groups: StorageKey, + metadata: store_v2::DatabaseMetadata, + metadata_key: StorageKey, + index_root: StorageKey, + row_root: StorageKey, + cursor_root: StorageKey, + path: PathBuf, + kdf_profile: KdfProfile, k_media: StorageKey, - k_local_metadata: StorageKey, - k_notes: StorageKey, - k_scheduled: StorageKey, - k_ephemeral: StorageKey, - k_devices: StorageKey, media_dir: PathBuf, media_limits: MediaLimits, // Prevents another Unix process from bypassing the pathname sidecar via a @@ -663,20 +756,19 @@ impl Store { rng: &mut impl CryptoRngCore, ) -> Result { let lock = acquire_store_lock(path)?; - let conn = Connection::open(path)?; - let database_lock = acquire_database_identity_lock(path)?; - if store_v2::is_inactive_migration_target(&conn)? { - return Err(StoreError::SchemaMismatch); + if path.exists() { + return Err(StoreError::NotAStore); } - conn.execute_batch(SCHEMA)?; - let existing: Option> = conn - .query_row("SELECT v FROM meta WHERE k = 'wrapped_sk'", [], |r| { - r.get(0) - }) - .optional()?; - if existing.is_some() { - return Err(StoreError::NotAStore); // refuse to clobber + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); } + options.open(path)?; + let conn = Connection::open(path)?; + let database_lock = acquire_database_identity_lock(path)?; let mut salt = [0u8; 16]; rng.fill_bytes(&mut salt); @@ -686,67 +778,68 @@ impl Store { let mut sk_bytes = Zeroizing::new([0u8; 32]); rng.fill_bytes(sk_bytes.as_mut()); let wrapped = kek_key.seal(WRAP_AD, sk_bytes.as_ref(), rng); + let master = StorageKey::from_bytes(*sk_bytes); + let metadata = store_v2::DatabaseMetadata::fresh(store_v2::random_database_id(rng)?); + let tx = Transaction::new_unchecked(&conn, TransactionBehavior::Immediate)?; + store_v2::create_schema(&tx)?; + store_v2::write_bootstrap(&tx, &salt, profile, &wrapped)?; + store_v2::write_metadata(&tx, &master, &metadata, rng)?; + tx.commit()?; + conn.pragma_update(None, "journal_mode", "WAL")?; + store_v2::configure_connection(&conn)?; + store_v2::protect_sqlite_files(path)?; - conn.execute("INSERT INTO meta (k, v) VALUES ('salt', ?1)", params![salt])?; - conn.execute( - "INSERT INTO meta (k, v) VALUES ('kdf', ?1)", - params![ - postcard::to_allocvec(&(profile.m_cost_kib, profile.t_cost, profile.p_cost)) - .map_err(|_| StoreError::Serialization)? - ], - )?; - conn.execute( - "INSERT INTO meta (k, v) VALUES ('wrapped_sk', ?1)", - params![wrapped], - )?; - - Self::with_master( - conn, - database_lock, - lock, - StorageKey::from_bytes(*sk_bytes), - path, - ) + Self::with_master(conn, database_lock, lock, master, metadata, profile, path) } /// Open and unlock an existing store. pub fn open(path: &Path, passphrase: &[u8]) -> Result { let lock = acquire_store_lock(path)?; + migration::recover_missing_source(path)?; + if !path.is_file() { + return Err(StoreError::NotAStore); + } let conn = Connection::open(path)?; let database_lock = acquire_database_identity_lock(path)?; - if store_v2::is_inactive_migration_target(&conn)? { - return Err(StoreError::SchemaMismatch); + if !store_v2::is_v2(&conn)? { + drop(conn); + return migration::migrate_legacy(path, passphrase, lock, database_lock); } - // Idempotent: also creates any table added since this store was — - // the only schema evolution so far is purely additive. - conn.execute_batch(SCHEMA)?; - let get = |k: &str| -> Result> { - conn.query_row("SELECT v FROM meta WHERE k = ?1", params![k], |r| r.get(0)) - .optional()? - .ok_or(StoreError::NotAStore) - }; - let salt: [u8; 16] = get("salt")?.try_into().map_err(|_| StoreError::NotAStore)?; - let (m, t, p): (u32, u32, u32) = - postcard::from_bytes(&get("kdf")?).map_err(|_| StoreError::NotAStore)?; - let wrapped = get("wrapped_sk")?; - - let profile = KdfProfile { - m_cost_kib: m, - t_cost: t, - p_cost: p, - }; + let store = Self::open_v2_with_parts(path, passphrase, conn, database_lock, lock, false)?; + migration::cleanup_completed_replacement(path)?; + backup::cleanup_completed_restore(path)?; + Ok(store) + } + + fn open_v2_with_parts( + path: &Path, + passphrase: &[u8], + conn: Connection, + database_lock: Option, + lock: File, + allow_incomplete: bool, + ) -> Result { + store_v2::validate_physical_schema(&conn)?; + store_v2::configure_connection(&conn)?; + let (salt, profile, wrapped) = store_v2::read_bootstrap(&conn)?; let kek = derive_kek(passphrase, &salt, profile)?; let kek_key = StorageKey::from_bytes(*kek); let sk_vec = Zeroizing::new(kek_key.open(WRAP_AD, &wrapped)?); // wrong passphrase fails here let sk_bytes: [u8; 32] = sk_vec[..].try_into().map_err(|_| StoreError::NotAStore)?; + let master = StorageKey::from_bytes(sk_bytes); + let metadata = store_v2::read_metadata(&conn, &master, allow_incomplete)?; + let store = Self::with_master(conn, database_lock, lock, master, metadata, profile, path)?; + store.validate_open_state()?; + store.conn.pragma_update(None, "journal_mode", "WAL")?; + store_v2::protect_sqlite_files(path)?; + Ok(store) + } - Self::with_master( - conn, - database_lock, - lock, - StorageKey::from_bytes(sk_bytes), - path, - ) + fn open_incomplete(path: &Path, passphrase: &[u8]) -> Result { + let lock = acquire_store_lock(path)?; + let conn = Connection::open(path)?; + let database_lock = acquire_database_identity_lock(path)?; + Self::open_v2_with_parts(path, passphrase, conn, database_lock, lock, true) } fn with_master( @@ -754,25 +847,22 @@ impl Store { database_lock: Option, lock: File, master: StorageKey, + metadata: store_v2::DatabaseMetadata, + profile: KdfProfile, path: &Path, ) -> Result { let media_dir = media::prepare_media_directory(path)?; + let keys = store_v2::derive_store_keys(&master); + let metadata_key = master.derive(b"KK-store-v2-metadata"); Ok(Self { - k_identity: master.derive(b"KK-store-identity"), - k_sessions: master.derive(b"KK-store-sessions"), - k_capabilities: master.derive(b"KK-store-capabilities"), - k_messages: master.derive(b"KK-store-messages"), - k_queue: master.derive(b"KK-store-queue"), - k_contacts: master.derive(b"KK-store-contacts"), - k_prekeys: master.derive(b"KK-store-prekeys"), - k_pending: master.derive(b"KK-store-pending"), - k_groups: master.derive(b"KK-store-groups"), - k_media: master.derive(b"KK-store-media"), - k_local_metadata: master.derive(b"KK-store-local-metadata"), - k_notes: master.derive(b"KK-store-notes"), - k_scheduled: master.derive(b"KK-store-scheduled"), - k_ephemeral: master.derive(b"KK-store-ephemeral"), - k_devices: master.derive(b"KK-store-devices"), + metadata, + metadata_key, + index_root: keys.index_root, + row_root: keys.row_root, + cursor_root: keys.cursor_root, + path: path.to_path_buf(), + kdf_profile: profile, + k_media: keys.media, media_dir, media_limits: MediaLimits::default(), conn, @@ -781,31 +871,136 @@ impl Store { }) } + fn validate_open_state(&self) -> Result<()> { + store_v2::validate_physical_schema(&self.conn)?; + self.validate_all_opaque_rows()?; + migration::validate_checkpoint_state(self)?; + self.validate_core_logical_rows()?; + self.validate_media_logical_rows()?; + self.validate_local_metadata_logical_rows()?; + self.validate_note_logical_rows()?; + self.validate_scheduled_logical_rows()?; + self.validate_ephemeral_logical_rows()?; + self.validate_device_logical_rows() + } + + fn validate_core_logical_rows(&self) -> Result<()> { + self.validate_rows::(|row| { + row.verify_key(&store_v2::SingletonKey)?; + row.verify_indexes(&store_v2::IndexKeys::none())?; + let bytes: [u8; 64] = row.payload[..] + .try_into() + .map_err(|_| StoreError::Serialization)?; + let _ = Identity::from_bytes(&bytes); + Ok(()) + })?; + self.validate_rows::(|row| { + let _ = store_v2::AccountKey::decode(&row.logical_key)?; + row.verify_indexes(&store_v2::IndexKeys::none())?; + let _: Session = decode_exact(&row.payload)?; + Ok(()) + })?; + self.validate_rows::(|row| { + let _ = store_v2::AccountKey::decode(&row.logical_key)?; + row.verify_indexes(&store_v2::IndexKeys::none())?; + let _ = CapabilityControl::decode(&row.payload)?; + Ok(()) + })?; + self.validate_rows::(|row| { + let record = self.decode_message_row_ref(row, None)?; + row.verify_indexes(&store_v2::IndexKeys::message( + &store_v2::ContentKey::new(record.id), + &store_v2::AccountKey::new(record.peer), + )) + })?; + self.validate_rows::(|row| { + let item = Self::decode_queue_item(&row.payload)?; + row.verify_indexes(&Self::queue_indexes(&item)) + })?; + self.validate_rows::(|row| { + let key = store_v2::ContentKey::decode(&row.logical_key)?; + row.verify_indexes(&store_v2::IndexKeys::none())?; + if row.payload.as_slice() != key.value() { + return Err(StoreError::LogicalKeyMismatch); + } + Ok(()) + })?; + self.validate_rows::(|row| { + let _ = store_v2::ContentKey::decode(&row.logical_key)?; + row.verify_indexes(&store_v2::IndexKeys::none())?; + let _: ([u8; 32], u64) = decode_exact(&row.payload)?; + Ok(()) + })?; + self.validate_rows::(|row| { + let record: ContactRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::AccountKey::new(record.peer))?; + row.verify_indexes(&store_v2::IndexKeys::none()) + })?; + self.validate_rows::(|row| { + row.verify_key(&store_v2::SingletonKey)?; + row.verify_indexes(&store_v2::IndexKeys::none()) + })?; + self.validate_rows::(|row| { + row.verify_indexes(&store_v2::IndexKeys::none())?; + let (encoded, _): (Vec, u64) = decode_exact(&row.payload)?; + let _ = Envelope::decode(&encoded)?; + Ok(()) + })?; + self.validate_rows::(|row| { + let key = store_v2::AccountKey::decode(&row.logical_key)?; + row.verify_indexes(&store_v2::IndexKeys::none())?; + if row.payload.as_slice() != key.value() { + return Err(StoreError::LogicalKeyMismatch); + } + Ok(()) + })?; + self.validate_rows::(|row| { + let record: GroupRecord = decode_exact(&row.payload)?; + let key = store_v2::GroupKey::decode(&row.logical_key)?; + if key.value() != &record.id { + return Err(StoreError::LogicalKeyMismatch); + } + row.verify_indexes(&store_v2::IndexKeys::none()) + })?; + self.validate_rows::(|row| { + let record: GroupAuthorityRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::GroupKey::new(record.group))?; + row.verify_indexes(&store_v2::IndexKeys::none()) + })?; + self.validate_rows::(|row| { + let key = store_v2::GroupMemberKey::decode(&row.logical_key)?; + row.verify_indexes(&store_v2::IndexKeys::group_chain(&store_v2::GroupKey::new( + *key.group(), + ))) + })?; + self.validate_rows::(|row| { + let record = self.decode_group_message_row(row)?; + row.verify_indexes(&store_v2::IndexKeys::group_message( + &store_v2::ContentKey::new(record.id), + &store_v2::GroupKey::new(record.group), + )) + }) + } + // ---- identity --------------------------------------------------------- /// Persist the device identity (sealed). pub fn put_identity(&self, id: &Identity, rng: &mut impl CryptoRngCore) -> Result<()> { - let sealed = self - .k_identity - .seal(b"identity", id.to_bytes().as_ref(), rng); - self.conn.execute( - "INSERT OR REPLACE INTO identity (id, blob) VALUES (1, ?1)", - params![sealed], - )?; - Ok(()) + self.put_equality::( + &store_v2::SingletonKey, + id.to_bytes().as_ref(), + store_v2::IndexKeys::none(), + rng, + ) } /// Load the device identity, if one was stored. pub fn get_identity(&self) -> Result> { - let sealed: Option> = self - .conn - .query_row("SELECT blob FROM identity WHERE id = 1", [], |r| r.get(0)) - .optional()?; - let Some(sealed) = sealed else { + let Some(row) = self.get_equality::(&store_v2::SingletonKey)? + else { return Ok(None); }; - let plain = Zeroizing::new(self.k_identity.open(b"identity", &sealed)?); - let bytes: [u8; 64] = plain[..] + let bytes: [u8; 64] = row.payload[..] .try_into() .map_err(|_| StoreError::Serialization)?; Ok(Some(Identity::from_bytes(&bytes))) @@ -820,28 +1015,21 @@ impl Store { session: &Session, rng: &mut impl CryptoRngCore, ) -> Result<()> { - let sealed = session.seal(&self.k_sessions, rng); - self.conn.execute( - "INSERT OR REPLACE INTO sessions (peer, blob) VALUES (?1, ?2)", - params![peer.as_slice(), sealed], - )?; - Ok(()) + let payload = + Zeroizing::new(postcard::to_allocvec(session).map_err(|_| StoreError::Serialization)?); + self.put_equality::( + &store_v2::AccountKey::new(*peer), + &payload, + store_v2::IndexKeys::none(), + rng, + ) } /// Load the session for a peer. pub fn get_session(&self, peer: &[u8; 32]) -> Result> { - let sealed: Option> = self - .conn - .query_row( - "SELECT blob FROM sessions WHERE peer = ?1", - params![peer.as_slice()], - |r| r.get(0), - ) - .optional()?; - match sealed { - Some(s) => Ok(Some(Session::unseal(&s, &self.k_sessions)?)), - None => Ok(None), - } + self.get_equality::(&store_v2::AccountKey::new(*peer))? + .map(|row| decode_exact(&row.payload)) + .transpose() } /// Atomically commit an accepted ordinary pairwise receive transition. @@ -864,57 +1052,67 @@ impl Store { return Err(StoreError::InvalidTransition); } if let Some(sequence) = plan.source_pending_sequence { - let sealed: Option> = self - .conn - .query_row( - "SELECT blob FROM pending WHERE seq = ?1", - params![sequence], - |row| row.get(0), - ) - .optional()?; - let Some(sealed) = sealed else { + let Some(row) = self.row_by_rowid::(sequence)? else { return Err(StoreError::InvalidTransition); }; - let plain = self.k_pending.open(b"pending", &sealed)?; - let (envelope, _): (Vec, u64) = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; + let (envelope, _): (Vec, u64) = decode_exact(&row.payload)?; if Envelope::decode(&envelope)?.content_id() != *plan.content_id { return Err(StoreError::InvalidTransition); } } - let sealed_session = plan.session.seal(&self.k_sessions, rng); - let sealed_message = if let Some(message) = plan.message { - let plain = postcard::to_allocvec(message).map_err(|_| StoreError::Serialization)?; - Some(self.k_messages.seal(b"message", &plain, rng)) + let session_payload = Zeroizing::new( + postcard::to_allocvec(plan.session).map_err(|_| StoreError::Serialization)?, + ); + let message_payload = if let Some(message) = plan.message { + Some(postcard::to_allocvec(message).map_err(|_| StoreError::Serialization)?) } else { None }; let replay = postcard::to_allocvec(&(*plan.peer_device, plan.received_at)) .map_err(|_| StoreError::Serialization)?; - let sealed_replay = self.k_queue.seal(b"receipt-replay", &replay, rng); let tx = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; let applied = (|| -> Result<()> { - tx.execute( - "INSERT OR REPLACE INTO sessions (peer, blob) VALUES (?1, ?2)", - params![plan.peer_device.as_slice(), sealed_session], + self.put_equality_on::( + &tx, + &store_v2::AccountKey::new(*plan.peer_device), + &session_payload, + store_v2::IndexKeys::none(), + rng, )?; - if let Some(sealed) = sealed_message { - tx.execute("INSERT INTO messages (blob) VALUES (?1)", params![sealed])?; + if let (Some(message), Some(payload)) = (plan.message, message_payload.as_deref()) { + self.append_on::( + &tx, + Some(&store_v2::MessageKey::new( + message.peer, + direction_code(message.direction), + message.id, + )), + payload, + store_v2::IndexKeys::message( + &store_v2::ContentKey::new(message.id), + &store_v2::AccountKey::new(message.peer), + ), + rng, + )?; } - tx.execute( - "INSERT INTO seen (id) VALUES (?1)", - params![plan.content_id.as_slice()], + self.put_equality_on::( + &tx, + &store_v2::ContentKey::new(*plan.content_id), + plan.content_id, + store_v2::IndexKeys::none(), + rng, )?; - tx.execute( - "INSERT OR REPLACE INTO receipt_replay (id, blob) VALUES (?1, ?2)", - params![plan.content_id.as_slice(), sealed_replay], + self.put_equality_on::( + &tx, + &store_v2::ContentKey::new(*plan.content_id), + &replay, + store_v2::IndexKeys::none(), + rng, )?; if let Some(sequence) = plan.source_pending_sequence { - let removed = - tx.execute("DELETE FROM pending WHERE seq = ?1", params![sequence])?; - if removed != 1 { + if !self.delete_rowid_on::(&tx, sequence)? { return Err(StoreError::InvalidTransition); } } @@ -935,10 +1133,7 @@ impl Store { /// Delete one exact physical-endpoint ratchet session. pub fn delete_session(&self, peer: &[u8; 32]) -> Result<()> { - self.conn.execute( - "DELETE FROM sessions WHERE peer = ?1", - params![peer.as_slice()], - )?; + self.delete_equality::(&store_v2::AccountKey::new(*peer))?; Ok(()) } @@ -952,42 +1147,26 @@ impl Store { capabilities: &CapabilityControl, rng: &mut impl CryptoRngCore, ) -> Result<()> { - let encoded = capabilities.encode()?; - let sealed = self.k_capabilities.seal(b"capability", &encoded, rng); - self.conn.execute( - "INSERT OR REPLACE INTO capabilities (peer, blob) VALUES (?1, ?2)", - params![peer.as_slice(), sealed], - )?; - Ok(()) + self.put_equality::( + &store_v2::AccountKey::new(*peer), + &capabilities.encode()?, + store_v2::IndexKeys::none(), + rng, + ) } /// Load the authenticated content-capability snapshot for a peer's /// current ratchet session. pub fn get_capabilities(&self, peer: &[u8; 32]) -> Result> { - let sealed: Option> = self - .conn - .query_row( - "SELECT blob FROM capabilities WHERE peer = ?1", - params![peer.as_slice()], - |row| row.get(0), - ) - .optional()?; - match sealed { - Some(sealed) => { - let plain = self.k_capabilities.open(b"capability", &sealed)?; - Ok(Some(CapabilityControl::decode(&plain)?)) - } - None => Ok(None), - } + self.get_equality::(&store_v2::AccountKey::new(*peer))? + .map(|row| CapabilityControl::decode(&row.payload).map_err(StoreError::from)) + .transpose() } /// Clear a peer capability snapshot when its ratchet session is reset or /// replaced. Capability state is re-creatable and never backed up. pub fn delete_capabilities(&self, peer: &[u8; 32]) -> Result<()> { - self.conn.execute( - "DELETE FROM capabilities WHERE peer = ?1", - params![peer.as_slice()], - )?; + self.delete_equality::(&store_v2::AccountKey::new(*peer))?; Ok(()) } @@ -996,49 +1175,112 @@ impl Store { /// Append a message record (sealed). pub fn put_message(&self, rec: &MessageRecord, rng: &mut impl CryptoRngCore) -> Result<()> { let plain = postcard::to_allocvec(rec).map_err(|_| StoreError::Serialization)?; - let sealed = self.k_messages.seal(b"message", &plain, rng); - self.conn - .execute("INSERT INTO messages (blob) VALUES (?1)", params![sealed])?; + let key = store_v2::MessageKey::new(rec.peer, direction_code(rec.direction), rec.id); + let indexes = store_v2::IndexKeys::message( + &store_v2::ContentKey::new(rec.id), + &store_v2::AccountKey::new(rec.peer), + ); + self.append::(&key, &plain, indexes, rng)?; Ok(()) } /// All messages for a peer, in insertion order. pub fn messages_with(&self, peer: &[u8; 32]) -> Result> { - Ok(self - .all_messages()? + let rows = self.rows_by_index::( + &store_v2::AccountKey::new(*peer), + )?; + rows.into_iter() + .map(|row| self.decode_message_row(row, Some(peer))) + .collect() + } + + /// Return one bounded pairwise history page without scanning another + /// conversation or decrypting rows beyond the requested page. + pub fn messages_page( + &self, + peer: &[u8; 32], + after: Option<&HistoryCursor>, + limit: usize, + ) -> Result { + if limit == 0 || limit > store_v2::MAX_PAGE_SIZE { + return Err(StoreError::RecordBounds); + } + let conversation = store_v2::AccountKey::new(*peer); + let after_rowid = after + .map(|cursor| { + self.decode_cursor::( + &conversation, + cursor.as_bytes(), + ) + }) + .transpose()?; + let rows = self.rows_by_index_after::( + &conversation, + after_rowid, + limit + 1, + )?; + let has_more = rows.len() > limit; + let selected = rows.into_iter().take(limit).collect::>(); + let next = if has_more { + selected.last().map(|row| { + HistoryCursor( + self.encode_cursor::(&conversation, row), + ) + }) + } else { + None + }; + let records = selected .into_iter() - .filter(|record| &record.peer == peer) - .collect()) + .map(|row| self.decode_message_row(row, Some(peer))) + .collect::>>()?; + Ok(MessagePage { records, next }) + } + + fn decode_message_row( + &self, + row: store_v2::RawRow, + expected_peer: Option<&[u8; 32]>, + ) -> Result { + self.decode_message_row_ref(&row, expected_peer) + } + + fn decode_message_row_ref( + &self, + row: &store_v2::RawRow, + expected_peer: Option<&[u8; 32]>, + ) -> Result { + let record: MessageRecord = decode_exact(&row.payload)?; + if expected_peer.is_some_and(|peer| peer != &record.peer) { + return Err(StoreError::LogicalKeyMismatch); + } + row.verify_key(&store_v2::MessageKey::new( + record.peer, + direction_code(record.direction), + record.id, + ))?; + Ok(record) } /// Replace the stored record with the same `id` as `rec`. Returns `true` - /// if a record was found and updated. (Records are sealed individually, - /// so lookup is a scan — fine at local-history scale.) + /// if a record was found and updated. pub fn update_message( &self, rec: &MessageRecord, rng: &mut impl CryptoRngCore, ) -> Result { - let mut stmt = self - .conn - .prepare("SELECT rowid_, blob FROM messages ORDER BY rowid_")?; - let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, Vec>(1)?)))?; - for row in rows { - let (rowid, sealed) = row?; - let plain = self.k_messages.open(b"message", &sealed)?; - let stored: MessageRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if stored.id == rec.id { - let plain = postcard::to_allocvec(rec).map_err(|_| StoreError::Serialization)?; - let sealed = self.k_messages.seal(b"message", &plain, rng); - self.conn.execute( - "UPDATE messages SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; - return Ok(true); - } + let id = store_v2::ContentKey::new(rec.id); + let Some(row) = self.row_by_unique::(&id)? else { + return Ok(false); + }; + let stored = self.decode_message_row_ref(&row, None)?; + if stored.peer != rec.peer || stored.direction != rec.direction { + return Err(StoreError::InvalidTransition); } - Ok(false) + let key = store_v2::MessageKey::new(rec.peer, direction_code(rec.direction), rec.id); + let indexes = store_v2::IndexKeys::message(&id, &store_v2::AccountKey::new(rec.peer)); + let plain = postcard::to_allocvec(rec).map_err(|_| StoreError::Serialization)?; + self.update_row::(&row.locator, &key, &plain, indexes, rng) } /// Delete one exact pairwise history row after an expiry tombstone is durable. @@ -1048,24 +1290,16 @@ impl Store { direction: Direction, id: &[u8; 16], ) -> Result { - let mut stmt = self - .conn - .prepare("SELECT rowid_, blob FROM messages ORDER BY rowid_")?; - let rows = stmt.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - })?; - for row in rows { - let (rowid, sealed) = row?; - let plain = self.k_messages.open(b"message", &sealed)?; - let record: MessageRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if &record.peer == peer && record.direction == direction && &record.id == id { - self.conn - .execute("DELETE FROM messages WHERE rowid_ = ?1", params![rowid])?; - return Ok(true); - } + let Some(row) = + self.row_by_unique::(&store_v2::ContentKey::new(*id))? + else { + return Ok(false); + }; + let record = self.decode_message_row_ref(&row, Some(peer))?; + if record.direction != direction || &record.id != id { + return Ok(false); } - Ok(false) + self.delete_row::(&row.locator) } // ---- contacts ---------------------------------------------------------- @@ -1073,53 +1307,42 @@ impl Store { /// Insert or replace a contact (sealed). pub fn put_contact(&self, rec: &ContactRecord, rng: &mut impl CryptoRngCore) -> Result<()> { let plain = postcard::to_allocvec(rec).map_err(|_| StoreError::Serialization)?; - let sealed = self.k_contacts.seal(b"contact", &plain, rng); - self.conn.execute( - "INSERT OR REPLACE INTO contacts (peer, blob) VALUES (?1, ?2)", - params![rec.peer.as_slice(), sealed], - )?; - Ok(()) + self.put_equality::( + &store_v2::AccountKey::new(rec.peer), + &plain, + store_v2::IndexKeys::none(), + rng, + ) } /// Load one contact. pub fn get_contact(&self, peer: &[u8; 32]) -> Result> { - let sealed: Option> = self - .conn - .query_row( - "SELECT blob FROM contacts WHERE peer = ?1", - params![peer.as_slice()], - |r| r.get(0), - ) - .optional()?; - match sealed { - Some(s) => { - let plain = self.k_contacts.open(b"contact", &s)?; - Ok(Some( - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?, - )) - } - None => Ok(None), - } + self.get_equality::(&store_v2::AccountKey::new(*peer))? + .map(|row| { + let record: ContactRecord = decode_exact(&row.payload)?; + if record.peer != *peer { + return Err(StoreError::LogicalKeyMismatch); + } + Ok(record) + }) + .transpose() } /// All contacts. pub fn contacts(&self) -> Result> { - let mut stmt = self.conn.prepare("SELECT blob FROM contacts")?; - let rows = stmt.query_map([], |r| r.get::<_, Vec>(0))?; - let mut out = Vec::new(); - for row in rows { - let plain = self.k_contacts.open(b"contact", &row?)?; - out.push(postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?); - } - Ok(out) + self.rows::()? + .into_iter() + .map(|row| { + let record: ContactRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::AccountKey::new(record.peer))?; + Ok(record) + }) + .collect() } /// Delete one exact sealed contact. Missing peers are an honest no-op. pub fn delete_contact(&self, peer: &[u8; 32]) -> Result { - Ok(self.conn.execute( - "DELETE FROM contacts WHERE peer = ?1", - params![peer.as_slice()], - )? == 1) + self.delete_equality::(&store_v2::AccountKey::new(*peer)) } // ---- own prekey secrets ------------------------------------------------- @@ -1127,140 +1350,42 @@ impl Store { /// Persist this device's prekey secrets as one opaque sealed blob (the /// runtime owns the serialization; the store interprets nothing). pub fn put_prekeys(&self, blob: &[u8], rng: &mut impl CryptoRngCore) -> Result<()> { - let sealed = self.k_prekeys.seal(b"prekeys", blob, rng); - self.conn.execute( - "INSERT OR REPLACE INTO prekeys (id, blob) VALUES (1, ?1)", - params![sealed], - )?; - Ok(()) + self.put_equality::( + &store_v2::SingletonKey, + blob, + store_v2::IndexKeys::none(), + rng, + ) } /// Load this device's prekey secrets blob, if stored. pub fn get_prekeys(&self) -> Result>>> { - let sealed: Option> = self - .conn - .query_row("SELECT blob FROM prekeys WHERE id = 1", [], |r| r.get(0)) - .optional()?; - match sealed { - Some(s) => Ok(Some(Zeroizing::new(self.k_prekeys.open(b"prekeys", &s)?))), - None => Ok(None), - } + Ok(self + .get_equality::(&store_v2::SingletonKey)? + .map(|row| row.payload)) } // ---- outbound queue --------------------------------------------------- /// Enqueue an envelope for delivery (sealed at rest; survives restarts). pub fn queue_push(&self, item: &QueueItem, rng: &mut impl CryptoRngCore) -> Result { - let envelope = item.envelope.try_encode()?; - let row = QueueRowV2 { - peer: item.peer, - msg_id: item.msg_id, - group_msg_id: item.group_msg_id, - class: item.class, - created_at: item.created_at, - attempts: item.attempts, - next_attempt_at: item.next_attempt_at, - envelope, - }; - let encoded = postcard::to_allocvec(&row).map_err(|_| StoreError::Serialization)?; - let mut plain = Vec::with_capacity(QUEUE_ROW_MAGIC_V2.len() + encoded.len()); - plain.extend_from_slice(QUEUE_ROW_MAGIC_V2); - plain.extend_from_slice(&encoded); - let sealed = self.k_queue.seal(b"queue", &plain, rng); - self.conn - .execute("INSERT INTO queue (blob) VALUES (?1)", params![sealed])?; - Ok(self.conn.last_insert_rowid()) + let payload = Self::encode_queue_item(item)?; + let row = + self.append_opaque::(&payload, Self::queue_indexes(item), rng)?; + Ok(row.rowid) } /// All queued items with their sequence numbers. pub fn queue_all(&self) -> Result> { - let mut stmt = self - .conn - .prepare("SELECT seq, blob FROM queue ORDER BY seq")?; - let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, Vec>(1)?)))?; - let mut out = Vec::new(); - for row in rows { - let (seq, sealed) = row?; - let plain = self.k_queue.open(b"queue", &sealed)?; - let ( - peer, - msg_id, - group_msg_id, - class, - created_at, - attempts, - next_attempt_at, - env_bytes, - ) = if let Some(encoded) = plain.strip_prefix(QUEUE_ROW_MAGIC_V2) { - let (row, remainder): (QueueRowV2, &[u8]) = - postcard::take_from_bytes(encoded).map_err(|_| StoreError::Serialization)?; - if !remainder.is_empty() { - return Err(StoreError::Serialization); - } - ( - row.peer, - row.msg_id, - row.group_msg_id, - row.class, - row.created_at, - row.attempts, - row.next_attempt_at, - row.envelope, - ) - } else if let Some(encoded) = plain.strip_prefix(QUEUE_ROW_MAGIC_V1) { - let (row, remainder): (QueueRowV1, &[u8]) = - postcard::take_from_bytes(encoded).map_err(|_| StoreError::Serialization)?; - if !remainder.is_empty() { - return Err(StoreError::Serialization); - } - ( - row.peer, - row.msg_id, - row.group_msg_id, - row.class, - 0, - 0, - 0, - row.envelope, - ) - } else { - let (legacy, remainder): (LegacyQueueRow, &[u8]) = - postcard::take_from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if !remainder.is_empty() { - return Err(StoreError::Serialization); - } - ( - legacy.0, - legacy.1, - legacy.2, - QueueClass::Normal, - 0, - 0, - 0, - legacy.3, - ) - }; - out.push(( - seq, - QueueItem { - peer, - msg_id, - group_msg_id, - class, - created_at, - attempts, - next_attempt_at, - envelope: Envelope::decode(&env_bytes)?, - }, - )); - } - Ok(out) + self.rows::()? + .into_iter() + .map(|row| Ok((row.rowid, Self::decode_queue_item(&row.payload)?))) + .collect() } /// Remove a delivered/acked envelope from the queue. pub fn queue_ack(&self, seq: i64) -> Result<()> { - self.conn - .execute("DELETE FROM queue WHERE seq = ?1", params![seq])?; + self.delete_rowid::(seq)?; Ok(()) } @@ -1272,40 +1397,32 @@ impl Store { item: &QueueItem, rng: &mut impl CryptoRngCore, ) -> Result<()> { - let envelope = item.envelope.try_encode()?; - let row = QueueRowV2 { - peer: item.peer, - msg_id: item.msg_id, - group_msg_id: item.group_msg_id, - class: item.class, - created_at: item.created_at, - attempts: item.attempts, - next_attempt_at: item.next_attempt_at, - envelope, + let Some(row) = self.row_by_rowid::(seq)? else { + return Ok(()); }; - let encoded = postcard::to_allocvec(&row).map_err(|_| StoreError::Serialization)?; - let mut plain = Vec::with_capacity(QUEUE_ROW_MAGIC_V2.len() + encoded.len()); - plain.extend_from_slice(QUEUE_ROW_MAGIC_V2); - plain.extend_from_slice(&encoded); - let sealed = self.k_queue.seal(b"queue", &plain, rng); - self.conn.execute( - "UPDATE queue SET blob = ?1 WHERE seq = ?2", - params![sealed, seq], + let locator: [u8; 16] = row + .locator + .as_slice() + .try_into() + .map_err(|_| StoreError::SchemaMismatch)?; + self.update_row::( + &row.locator, + &store_v2::OpaqueRowKey::from_locator(locator), + &Self::encode_queue_item(item)?, + Self::queue_indexes(item), + rng, )?; Ok(()) } /// Remove every queued envelope addressed to one revoked physical endpoint. pub fn queue_remove_peer(&self, peer: &[u8; 32]) -> Result { - let sequences: Vec = self - .queue_all()? - .into_iter() - .filter_map(|(seq, item)| (&item.peer == peer).then_some(seq)) - .collect(); - for sequence in &sequences { - self.queue_ack(*sequence)?; + let rows = + self.rows_by_index::(&store_v2::AccountKey::new(*peer))?; + for row in &rows { + self.delete_row::(&row.locator)?; } - Ok(sequences.len()) + Ok(rows.len()) } /// Retarget durable queue ownership after a legacy endpoint is bound to @@ -1316,58 +1433,61 @@ impl Store { new_peer: &[u8; 32], rng: &mut impl CryptoRngCore, ) -> Result { - let rows: Vec<(i64, QueueItem)> = self - .queue_all()? - .into_iter() - .filter(|(_, item)| &item.peer == old_peer) - .collect(); - for (sequence, mut item) in rows.iter().cloned() { - self.queue_ack(sequence)?; + let rows = + self.rows_by_index::(&store_v2::AccountKey::new(*old_peer))?; + for row in &rows { + let mut item = Self::decode_queue_item(&row.payload)?; + if item.peer != *old_peer { + return Err(StoreError::LogicalKeyMismatch); + } item.peer = *new_peer; - self.queue_push(&item, rng)?; + let locator: [u8; 16] = row + .locator + .as_slice() + .try_into() + .map_err(|_| StoreError::SchemaMismatch)?; + self.update_row::( + &row.locator, + &store_v2::OpaqueRowKey::from_locator(locator), + &Self::encode_queue_item(&item)?, + Self::queue_indexes(&item), + rng, + )?; } Ok(rows.len()) } /// Remove every queued envelope associated with one expired pairwise message. pub fn queue_remove_message(&self, id: &[u8; 16]) -> Result { - let sequences: Vec = self - .queue_all()? - .into_iter() - .filter_map(|(seq, item)| (item.msg_id.as_ref() == Some(id)).then_some(seq)) - .collect(); - for sequence in &sequences { - self.queue_ack(*sequence)?; + let rows = + self.rows_by_index::(&store_v2::ContentKey::new(*id))?; + for row in &rows { + self.delete_row::(&row.locator)?; } - Ok(sequences.len()) + Ok(rows.len()) } /// Remove every queued member copy associated with one expired group message. pub fn queue_remove_group_message(&self, id: &[u8; 16]) -> Result { - let sequences: Vec = self - .queue_all()? - .into_iter() - .filter_map(|(seq, item)| (item.group_msg_id.as_ref() == Some(id)).then_some(seq)) - .collect(); - for sequence in &sequences { - self.queue_ack(*sequence)?; + let rows = self + .rows_by_index::(&store_v2::ContentKey::new(*id))?; + for row in &rows { + self.delete_row::(&row.locator)?; } - Ok(sequences.len()) + Ok(rows.len()) } /// Remove queued copies of one exact sealed envelope after its encrypted /// end-to-end receipt returns. Matching the content id keeps other linked /// devices' copies of the same logical message independently retryable. pub fn queue_remove_envelope(&self, content_id: &[u8; 16]) -> Result { - let sequences: Vec = self - .queue_all()? - .into_iter() - .filter_map(|(seq, item)| (item.envelope.content_id() == *content_id).then_some(seq)) - .collect(); - for sequence in &sequences { - self.queue_ack(*sequence)?; + let rows = self.rows_by_index::( + &store_v2::ContentKey::new(*content_id), + )?; + for row in &rows { + self.delete_row::(&row.locator)?; } - Ok(sequences.len()) + Ok(rows.len()) } // ---- inbound pending (envelopes that cannot be processed yet) --------- @@ -1385,29 +1505,29 @@ impl Store { let encoded = env.try_encode()?; let plain = postcard::to_allocvec(&(encoded, first_seen)).map_err(|_| StoreError::Serialization)?; - let sealed = self.k_pending.seal(b"pending", &plain, rng); - if sealed.len() > MAX_PENDING_BYTES { + if plain.len() > MAX_PENDING_BYTES { return Err(StoreError::PendingQuota); } let tx = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; - let (count, bytes): (i64, i64) = tx.query_row( - "SELECT COUNT(*), COALESCE(SUM(length(blob)), 0) FROM pending", - [], - |row| Ok((row.get(0)?, row.get(1)?)), + let count = usize::try_from(self.count_rows_on::(&tx)?) + .map_err(|_| StoreError::Serialization)?; + if count >= MAX_PENDING_ENVELOPES { + tx.rollback()?; + return Err(StoreError::PendingQuota); + } + let row = self.append_on::( + &tx, + None, + &plain, + store_v2::IndexKeys::none(), + rng, )?; - let count = usize::try_from(count).map_err(|_| StoreError::Serialization)?; - let bytes = usize::try_from(bytes).map_err(|_| StoreError::Serialization)?; - if count >= MAX_PENDING_ENVELOPES - || bytes - .checked_add(sealed.len()) - .is_none_or(|total| total > MAX_PENDING_BYTES) - { + if self.sealed_bytes_on::(&tx)? > MAX_PENDING_BYTES as u64 { tx.rollback()?; return Err(StoreError::PendingQuota); } - tx.execute("INSERT INTO pending (blob) VALUES (?1)", params![sealed])?; - let sequence = tx.last_insert_rowid(); + let sequence = row.rowid; tx.commit()?; Ok(sequence) } @@ -1419,19 +1539,13 @@ impl Store { /// after the envelope is consumed or has expired. This gives deferred /// receive processing at-least-once crash semantics. pub fn pending_all(&self) -> Result> { - let mut stmt = self - .conn - .prepare("SELECT seq, blob FROM pending ORDER BY seq")?; - let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, Vec>(1)?)))?; - let mut out = Vec::new(); - for row in rows { - let (sequence, sealed) = row?; - let plain = self.k_pending.open(b"pending", &sealed)?; - let (env_bytes, first_seen): (Vec, u64) = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - out.push((sequence, Envelope::decode(&env_bytes)?, first_seen)); - } - Ok(out) + self.rows::()? + .into_iter() + .map(|row| { + let (env_bytes, first_seen): (Vec, u64) = decode_exact(&row.payload)?; + Ok((row.rowid, Envelope::decode(&env_bytes)?, first_seen)) + }) + .collect() } /// Acknowledge one consumed or expired inbound envelope. @@ -1440,54 +1554,126 @@ impl Store { /// not-yet-visited rows remain durable if processing returns an error or /// the process stops between envelopes. pub fn pending_ack(&self, sequence: i64) -> Result<()> { - self.conn - .execute("DELETE FROM pending WHERE seq = ?1", params![sequence])?; + self.delete_rowid::(sequence)?; Ok(()) } + fn encode_queue_item(item: &QueueItem) -> Result> { + let row = QueueRowV2 { + peer: item.peer, + msg_id: item.msg_id, + group_msg_id: item.group_msg_id, + class: item.class, + created_at: item.created_at, + attempts: item.attempts, + next_attempt_at: item.next_attempt_at, + envelope: item.envelope.try_encode()?, + }; + let encoded = postcard::to_allocvec(&row).map_err(|_| StoreError::Serialization)?; + let mut plain = Vec::with_capacity(QUEUE_ROW_MAGIC_V2.len() + encoded.len()); + plain.extend_from_slice(QUEUE_ROW_MAGIC_V2); + plain.extend_from_slice(&encoded); + Ok(plain) + } + + fn decode_queue_item(plain: &[u8]) -> Result { + let (peer, msg_id, group_msg_id, class, created_at, attempts, next_attempt_at, envelope) = + if let Some(encoded) = plain.strip_prefix(QUEUE_ROW_MAGIC_V2) { + let row: QueueRowV2 = decode_exact(encoded)?; + ( + row.peer, + row.msg_id, + row.group_msg_id, + row.class, + row.created_at, + row.attempts, + row.next_attempt_at, + row.envelope, + ) + } else if let Some(encoded) = plain.strip_prefix(QUEUE_ROW_MAGIC_V1) { + let row: QueueRowV1 = decode_exact(encoded)?; + ( + row.peer, + row.msg_id, + row.group_msg_id, + row.class, + 0, + 0, + 0, + row.envelope, + ) + } else { + let legacy: LegacyQueueRow = decode_exact(plain)?; + ( + legacy.0, + legacy.1, + legacy.2, + QueueClass::Normal, + 0, + 0, + 0, + legacy.3, + ) + }; + Ok(QueueItem { + peer, + msg_id, + group_msg_id, + class, + created_at, + attempts, + next_attempt_at, + envelope: Envelope::decode(&envelope)?, + }) + } + + fn queue_indexes(item: &QueueItem) -> store_v2::IndexKeys { + let message = item.msg_id.map(store_v2::ContentKey::new); + let group_message = item.group_msg_id.map(store_v2::ContentKey::new); + store_v2::IndexKeys::queue( + &store_v2::AccountKey::new(item.peer), + message.as_ref(), + group_message.as_ref(), + &store_v2::ContentKey::new(item.envelope.content_id()), + ) + } + // ---- groups (ADR-0012) -------------------------------------------------- /// Insert or replace a group (sealed). pub fn put_group(&self, rec: &GroupRecord, rng: &mut impl CryptoRngCore) -> Result<()> { let plain = Zeroizing::new(postcard::to_allocvec(rec).map_err(|_| StoreError::Serialization)?); - let sealed = self.k_groups.seal(b"group", &plain, rng); - self.conn.execute( - "INSERT OR REPLACE INTO groups (gid, blob) VALUES (?1, ?2)", - params![rec.id.as_slice(), sealed], + self.put_equality::( + &store_v2::GroupKey::new(rec.id), + &plain, + store_v2::IndexKeys::none(), + rng, )?; Ok(()) } /// Load one group. pub fn get_group(&self, id: &[u8; 32]) -> Result> { - let sealed: Option> = self - .conn - .query_row( - "SELECT blob FROM groups WHERE gid = ?1", - params![id.as_slice()], - |r| r.get(0), - ) - .optional()?; - match sealed { - Some(s) => { - let plain = Zeroizing::new(self.k_groups.open(b"group", &s)?); - Ok(Some( - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?, - )) - } - None => Ok(None), + let key = store_v2::GroupKey::new(*id); + let Some(row) = self.get_equality::(&key)? else { + return Ok(None); + }; + row.verify_key(&key)?; + let record: GroupRecord = decode_exact(&row.payload)?; + if record.id != *id { + return Err(StoreError::LogicalKeyMismatch); } + Ok(Some(record)) } /// All groups. pub fn groups(&self) -> Result> { - let mut stmt = self.conn.prepare("SELECT blob FROM groups")?; - let rows = stmt.query_map([], |r| r.get::<_, Vec>(0))?; let mut out = Vec::new(); - for row in rows { - let plain = Zeroizing::new(self.k_groups.open(b"group", &row?)?); - out.push(postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?); + for row in self.rows::()? { + let record: GroupRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::GroupKey::new(record.id))?; + out.push(record); } Ok(out) } @@ -1495,16 +1681,15 @@ impl Store { /// Remove a group and every receiving chain under it (leaving keeps the /// message history — that is this device's data). pub fn delete_group(&self, id: &[u8; 32]) -> Result<()> { - self.conn - .execute("DELETE FROM groups WHERE gid = ?1", params![id.as_slice()])?; - self.conn.execute( - "DELETE FROM group_chains WHERE gid = ?1", - params![id.as_slice()], - )?; - self.conn.execute( - "DELETE FROM group_authority WHERE gid = ?1", - params![id.as_slice()], - )?; + let group = store_v2::GroupKey::new(*id); + let chains = self.rows_by_index::(&group)?; + let tx = self.conn.unchecked_transaction()?; + self.delete_equality_on::(&tx, &group)?; + self.delete_equality_on::(&tx, &group)?; + for chain in chains { + self.delete_rowid_on::(&tx, chain.rowid)?; + } + tx.commit()?; Ok(()) } @@ -1516,43 +1701,36 @@ impl Store { ) -> Result<()> { let plain = Zeroizing::new(postcard::to_allocvec(rec).map_err(|_| StoreError::Serialization)?); - let sealed = self.k_groups.seal(b"group-authority", &plain, rng); - self.conn.execute( - "INSERT OR REPLACE INTO group_authority (gid, blob) VALUES (?1, ?2)", - params![rec.group.as_slice(), sealed], + self.put_equality::( + &store_v2::GroupKey::new(rec.group), + &plain, + store_v2::IndexKeys::none(), + rng, )?; Ok(()) } /// Load one group's sealed signed authority state. pub fn get_group_authority(&self, group: &[u8; 32]) -> Result> { - let sealed: Option> = self - .conn - .query_row( - "SELECT blob FROM group_authority WHERE gid = ?1", - params![group.as_slice()], - |row| row.get(0), - ) - .optional()?; - match sealed { - Some(sealed) => { - let plain = Zeroizing::new(self.k_groups.open(b"group-authority", &sealed)?); - Ok(Some( - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?, - )) - } - None => Ok(None), + let key = store_v2::GroupKey::new(*group); + let Some(row) = self.get_equality::(&key)? else { + return Ok(None); + }; + row.verify_key(&key)?; + let record: GroupAuthorityRecord = decode_exact(&row.payload)?; + if record.group != *group { + return Err(StoreError::LogicalKeyMismatch); } + Ok(Some(record)) } /// All sealed C6 authority records for backup and audit. pub fn group_authorities(&self) -> Result> { - let mut stmt = self.conn.prepare("SELECT blob FROM group_authority")?; - let rows = stmt.query_map([], |row| row.get::<_, Vec>(0))?; let mut out = Vec::new(); - for row in rows { - let plain = Zeroizing::new(self.k_groups.open(b"group-authority", &row?)?); - out.push(postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?); + for row in self.rows::()? { + let record: GroupAuthorityRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::GroupKey::new(record.group))?; + out.push(record); } Ok(out) } @@ -1566,10 +1744,12 @@ impl Store { blob: &[u8], rng: &mut impl CryptoRngCore, ) -> Result<()> { - let sealed = self.k_groups.seal(b"group-chain", blob, rng); - self.conn.execute( - "INSERT OR REPLACE INTO group_chains (gid, peer, blob) VALUES (?1, ?2, ?3)", - params![group.as_slice(), peer.as_slice(), sealed], + let key = store_v2::GroupMemberKey::new(*group, *peer); + self.put_equality::( + &key, + blob, + store_v2::IndexKeys::group_chain(&store_v2::GroupKey::new(*group)), + rng, )?; Ok(()) } @@ -1580,38 +1760,24 @@ impl Store { group: &[u8; 32], peer: &[u8; 32], ) -> Result>>> { - let sealed: Option> = self - .conn - .query_row( - "SELECT blob FROM group_chains WHERE gid = ?1 AND peer = ?2", - params![group.as_slice(), peer.as_slice()], - |r| r.get(0), - ) - .optional()?; - match sealed { - Some(s) => Ok(Some(Zeroizing::new( - self.k_groups.open(b"group-chain", &s)?, - ))), - None => Ok(None), - } + let key = store_v2::GroupMemberKey::new(*group, *peer); + let Some(row) = self.get_equality::(&key)? else { + return Ok(None); + }; + row.verify_key(&key)?; + Ok(Some(row.payload)) } /// All receiving chains for a group, as `(peer, blob)`. pub fn group_chains(&self, group: &[u8; 32]) -> Result> { - let mut stmt = self - .conn - .prepare("SELECT peer, blob FROM group_chains WHERE gid = ?1")?; - let rows = stmt.query_map(params![group.as_slice()], |r| { - Ok((r.get::<_, Vec>(0)?, r.get::<_, Vec>(1)?)) - })?; + let group_key = store_v2::GroupKey::new(*group); let mut out = Vec::new(); - for row in rows { - let (peer, sealed) = row?; - let peer: [u8; 32] = peer.try_into().map_err(|_| StoreError::Serialization)?; - out.push(( - peer, - Zeroizing::new(self.k_groups.open(b"group-chain", &sealed)?), - )); + for row in self.rows_by_index::(&group_key)? { + let key = store_v2::GroupMemberKey::decode(&row.logical_key)?; + if key.group() != group { + return Err(StoreError::LogicalKeyMismatch); + } + out.push((*key.peer(), row.payload)); } Ok(out) } @@ -1619,10 +1785,9 @@ impl Store { /// Drop one member's receiving chain (they were removed or rotated to a /// new chain that replaces this one). pub fn delete_group_chain(&self, group: &[u8; 32], peer: &[u8; 32]) -> Result<()> { - self.conn.execute( - "DELETE FROM group_chains WHERE gid = ?1 AND peer = ?2", - params![group.as_slice(), peer.as_slice()], - )?; + self.delete_equality::(&store_v2::GroupMemberKey::new( + *group, *peer, + ))?; Ok(()) } @@ -1632,10 +1797,21 @@ impl Store { rec: &GroupMessageRecord, rng: &mut impl CryptoRngCore, ) -> Result<()> { - let plain = postcard::to_allocvec(rec).map_err(|_| StoreError::Serialization)?; - let sealed = self.k_groups.seal(b"group-msg", &plain, rng); - self.conn - .execute("INSERT INTO group_msgs (blob) VALUES (?1)", params![sealed])?; + let plain = + Zeroizing::new(postcard::to_allocvec(rec).map_err(|_| StoreError::Serialization)?); + let id = store_v2::ContentKey::new(rec.id); + let group = store_v2::GroupKey::new(rec.group); + self.append::( + &store_v2::GroupMessageKey::new( + rec.group, + rec.sender, + direction_code(rec.direction), + rec.id, + ), + &plain, + store_v2::IndexKeys::group_message(&id, &group), + rng, + )?; Ok(()) } @@ -1646,26 +1822,26 @@ impl Store { rec: &GroupMessageRecord, rng: &mut impl CryptoRngCore, ) -> Result { - let mut stmt = self - .conn - .prepare("SELECT rowid_, blob FROM group_msgs ORDER BY rowid_")?; - let rows = stmt.query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, Vec>(1)?)))?; - for row in rows { - let (rowid, sealed) = row?; - let plain = self.k_groups.open(b"group-msg", &sealed)?; - let stored: GroupMessageRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if stored.id == rec.id { - let plain = postcard::to_allocvec(rec).map_err(|_| StoreError::Serialization)?; - let sealed = self.k_groups.seal(b"group-msg", &plain, rng); - self.conn.execute( - "UPDATE group_msgs SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; - return Ok(true); - } - } - Ok(false) + let id = store_v2::ContentKey::new(rec.id); + let Some(row) = self.row_by_unique::(&id)? else { + return Ok(false); + }; + let expected = store_v2::GroupMessageKey::new( + rec.group, + rec.sender, + direction_code(rec.direction), + rec.id, + ); + row.verify_key(&expected)?; + let plain = + Zeroizing::new(postcard::to_allocvec(rec).map_err(|_| StoreError::Serialization)?); + self.update_row::( + &row.locator, + &expected, + &plain, + store_v2::IndexKeys::group_message(&id, &store_v2::GroupKey::new(rec.group)), + rng, + ) } /// Delete one exact group history row after an expiry tombstone is durable. @@ -1675,73 +1851,115 @@ impl Store { sender: &[u8; 32], id: &[u8; 16], ) -> Result { - let mut stmt = self - .conn - .prepare("SELECT rowid_, blob FROM group_msgs ORDER BY rowid_")?; - let rows = stmt.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - })?; - for row in rows { - let (rowid, sealed) = row?; - let plain = self.k_groups.open(b"group-msg", &sealed)?; - let record: GroupMessageRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if &record.group == group && &record.sender == sender && &record.id == id { - self.conn - .execute("DELETE FROM group_msgs WHERE rowid_ = ?1", params![rowid])?; - return Ok(true); - } + let id_key = store_v2::ContentKey::new(*id); + let Some(row) = self.row_by_unique::(&id_key)? else { + return Ok(false); + }; + let record = self.decode_group_message_row(&row)?; + if record.group != *group || record.sender != *sender || record.id != *id { + return Err(StoreError::LogicalKeyMismatch); } - Ok(false) + self.delete_row::(&row.locator) } /// All messages for a group, in insertion order. pub fn group_messages(&self, group: &[u8; 32]) -> Result> { - Ok(self - .all_group_messages()? - .into_iter() - .filter(|r| &r.group == group) - .collect()) + let mut out = Vec::new(); + for row in self.rows_by_index::( + &store_v2::GroupKey::new(*group), + )? { + out.push(self.decode_group_message_row(&row)?); + } + Ok(out) + } + + /// One bounded page of group history in stable insertion order. + pub fn group_messages_page( + &self, + group: &[u8; 32], + after: Option<&HistoryCursor>, + limit: usize, + ) -> Result { + if limit == 0 || limit > store_v2::MAX_PAGE_SIZE { + return Err(StoreError::RecordBounds); + } + let group_key = store_v2::GroupKey::new(*group); + let after_rowid = after + .map(|cursor| { + self.decode_cursor::( + &group_key, + cursor.as_bytes(), + ) + }) + .transpose()?; + let rows = self.rows_by_index_after::( + &group_key, + after_rowid, + limit, + )?; + let next = rows.last().map(|row| { + HistoryCursor( + self.encode_cursor::(&group_key, row), + ) + }); + let records = rows + .iter() + .map(|row| self.decode_group_message_row(row)) + .collect::>>()?; + Ok(GroupMessagePage { records, next }) } /// Every stored group message across all groups, in insertion order /// (receipt application scans this; local history stays small). pub fn all_group_messages(&self) -> Result> { - let mut stmt = self - .conn - .prepare("SELECT blob FROM group_msgs ORDER BY rowid_")?; - let rows = stmt.query_map([], |r| r.get::<_, Vec>(0))?; let mut out = Vec::new(); - for row in rows { - let plain = self.k_groups.open(b"group-msg", &row?)?; - out.push(postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?); + for row in self.rows::()? { + out.push(self.decode_group_message_row(&row)?); } Ok(out) } + fn decode_group_message_row(&self, row: &store_v2::RawRow) -> Result { + let record: GroupMessageRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::GroupMessageKey::new( + record.group, + record.sender, + direction_code(record.direction), + record.id, + ))?; + Ok(record) + } + // ---- dedup ------------------------------------------------------------ /// Record an envelope content id; returns `true` if it was new /// (multipath duplicates return `false` and must be dropped). pub fn mark_seen(&self, content_id: &[u8; 16]) -> Result { - let n = self.conn.execute( - "INSERT OR IGNORE INTO seen (id) VALUES (?1)", - params![content_id.as_slice()], + let key = store_v2::ContentKey::new(*content_id); + if self.get_equality::(&key)?.is_some() { + return Ok(false); + } + let mut rng = rand_core::OsRng; + self.put_equality::( + &key, + content_id, + store_v2::IndexKeys::none(), + &mut rng, )?; - Ok(n == 1) + Ok(true) } /// Has this envelope content id been consumed before? pub fn is_seen(&self, content_id: &[u8; 16]) -> Result { - let found: Option = self - .conn - .query_row( - "SELECT 1 FROM seen WHERE id = ?1", - params![content_id.as_slice()], - |r| r.get(0), - ) - .optional()?; - Ok(found.is_some()) + let key = store_v2::ContentKey::new(*content_id); + let Some(row) = self.get_equality::(&key)? else { + return Ok(false); + }; + row.verify_key(&key)?; + if row.payload.as_slice() != content_id { + return Err(StoreError::LogicalKeyMismatch); + } + Ok(true) } /// Remember where an accepted envelope's encrypted receipt must return, @@ -1756,55 +1974,41 @@ impl Store { ) -> Result<()> { let encoded = postcard::to_allocvec(&(*peer, received_at)).map_err(|_| StoreError::Serialization)?; - let sealed = self.k_queue.seal(b"receipt-replay", &encoded, rng); - self.conn.execute( - "INSERT OR REPLACE INTO receipt_replay (id, blob) VALUES (?1, ?2)", - params![id.as_slice(), sealed], + self.put_equality::( + &store_v2::ContentKey::new(*id), + &encoded, + store_v2::IndexKeys::none(), + rng, )?; Ok(()) } /// Return the physical sender route for a previously accepted envelope. pub fn receipt_replay_peer(&self, id: &[u8; 16]) -> Result> { - let sealed: Option> = self - .conn - .query_row( - "SELECT blob FROM receipt_replay WHERE id = ?1", - params![id.as_slice()], - |row| row.get(0), - ) - .optional()?; - let Some(sealed) = sealed else { + let key = store_v2::ContentKey::new(*id); + let Some(row) = self.get_equality::(&key)? else { return Ok(None); }; - let plain = self.k_queue.open(b"receipt-replay", &sealed)?; - let (peer, _): ([u8; 32], u64) = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; + row.verify_key(&key)?; + let (peer, _): ([u8; 32], u64) = decode_exact(&row.payload)?; Ok(Some(peer)) } /// Remove duplicate-receipt routes older than the endpoint delivery /// window. Seen ids remain independent and keep deduplication durable. pub fn sweep_receipt_replay(&self, cutoff: u64) -> Result { - let mut stmt = self.conn.prepare("SELECT id, blob FROM receipt_replay")?; - let rows = stmt.query_map([], |row| { - Ok((row.get::<_, Vec>(0)?, row.get::<_, Vec>(1)?)) - })?; let mut expired = Vec::new(); - for row in rows { - let (id, sealed) = row?; - let plain = self.k_queue.open(b"receipt-replay", &sealed)?; - let (_, received_at): ([u8; 32], u64) = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; + for row in self.rows::()? { + let (_, received_at): ([u8; 32], u64) = decode_exact(&row.payload)?; if received_at <= cutoff { - expired.push(id); + expired.push(row.rowid); } } - drop(stmt); - for id in &expired { - self.conn - .execute("DELETE FROM receipt_replay WHERE id = ?1", params![id])?; + let tx = self.conn.unchecked_transaction()?; + for rowid in &expired { + self.delete_rowid_on::(&tx, *rowid)?; } + tx.commit()?; Ok(expired.len()) } } @@ -1855,10 +2059,22 @@ mod queue_tests { envelope.encode(), )) .unwrap(); - let sealed = store.k_queue.seal(b"queue", &legacy, &mut rng); + let legacy_item = QueueItem { + peer: [4; 32], + msg_id: None, + group_msg_id: None, + class: QueueClass::Normal, + created_at: 0, + attempts: 0, + next_attempt_at: 0, + envelope: envelope.clone(), + }; store - .conn - .execute("INSERT INTO queue (blob) VALUES (?1)", params![sealed]) + .append_opaque::( + &legacy, + Store::queue_indexes(&legacy_item), + &mut rng, + ) .unwrap(); let rows = store.queue_all().unwrap(); @@ -1969,6 +2185,22 @@ mod queue_tests { assert!(reopened.pending_all().unwrap().is_empty()); } + #[test] + fn wrong_passphrase_does_not_rewrite_the_database() { + let mut rng = StdRng::seed_from_u64(0xbad5ea); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("wrong-pass.db"); + let store = Store::create(&path, b"right", TEST_KDF, &mut rng).unwrap(); + drop(store); + let before = std::fs::read(&path).unwrap(); + + assert!(matches!( + Store::open(&path, b"wrong"), + Err(StoreError::Crypto(_)) + )); + assert_eq!(std::fs::read(&path).unwrap(), before); + } + #[test] fn pending_inbox_enforces_item_and_sealed_byte_quotas() { let mut rng = StdRng::seed_from_u64(0x1b1); @@ -1991,8 +2223,12 @@ mod queue_tests { let tx = Transaction::new_unchecked(&store.conn, TransactionBehavior::Immediate).unwrap(); for _ in 0..MAX_PENDING_ENVELOPES { - tx.execute("INSERT INTO pending (blob) VALUES (zeroblob(1))", []) - .unwrap(); + tx.execute( + "INSERT INTO store_records (table_domain, locator, blob) + VALUES (10, randomblob(16), zeroblob(1))", + [], + ) + .unwrap(); } tx.commit().unwrap(); assert!(matches!( @@ -2000,11 +2236,15 @@ mod queue_tests { Err(StoreError::PendingQuota) )); - store.conn.execute("DELETE FROM pending", []).unwrap(); + store + .conn + .execute("DELETE FROM store_records WHERE table_domain = 10", []) + .unwrap(); store .conn .execute( - "INSERT INTO pending (blob) VALUES (zeroblob(?1))", + "INSERT INTO store_records (table_domain, locator, blob) + VALUES (10, randomblob(16), zeroblob(?1))", params![MAX_PENDING_BYTES as i64], ) .unwrap(); @@ -2075,7 +2315,8 @@ mod queue_tests { .conn .execute_batch( "CREATE TRIGGER fail_pairwise_receive - BEFORE INSERT ON seen + BEFORE INSERT ON store_records + WHEN NEW.table_domain = 6 BEGIN SELECT RAISE(ABORT, 'injected receive failure'); END;", diff --git a/crates/kult-store/src/local_metadata.rs b/crates/kult-store/src/local_metadata.rs index f664dd3..195265e 100644 --- a/crates/kult-store/src/local_metadata.rs +++ b/crates/kult-store/src/local_metadata.rs @@ -2,20 +2,20 @@ //! //! These records never enter envelopes, DHT records, group state, or transport //! hints. The SQLite table contains only an insertion-order row id and one -//! independently sealed blob, so copied databases reveal neither record keys -//! nor organization relationships. +//! independently row-bound sealed blob plus keyed opaque indexes, so copied +//! databases reveal neither record keys nor organization relationships. use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; use rand_core::CryptoRngCore; -use rusqlite::params; +use rusqlite::Connection; use serde::{Deserialize, Serialize}; +use zeroize::Zeroizing; -use crate::{Result, Store, StoreError}; +use crate::{decode_exact, store_v2, Result, Store, StoreError}; const RECORD_MAGIC_V1: &[u8; 4] = b"KLM1"; -const RECORD_AD: &[u8] = b"local-metadata"; /// Maximum UTF-8 bytes in a folder name, label name, color token, media type, /// preference key, or similar local-metadata string. @@ -620,24 +620,7 @@ impl Store { if let LocalMetadataRecord::CustomIcon(icon) = record { self.validate_custom_icon_quota(icon)?; } - let key = record.key(); - let existing = self - .local_metadata_rows()? - .into_iter() - .find_map(|(rowid, stored)| (stored.key() == key).then_some(rowid)); - let sealed = self.seal_local_metadata(record, rng)?; - if let Some(rowid) = existing { - self.conn.execute( - "UPDATE local_metadata SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; - } else { - self.conn.execute( - "INSERT INTO local_metadata (blob) VALUES (?1)", - params![sealed], - )?; - } - Ok(()) + self.put_local_metadata_on(&self.conn, record, rng) } /// Read one local metadata record by its stable logical key. @@ -645,10 +628,16 @@ impl Store { &self, key: &LocalMetadataKey, ) -> Result> { - Ok(self - .local_metadata_rows()? - .into_iter() - .find_map(|(_, record)| (&record.key() == key).then_some(record))) + let physical_key = local_metadata_key(key)?; + let Some(row) = self.get_equality::(&physical_key)? else { + return Ok(None); + }; + row.verify_key(&physical_key)?; + let record = decode_local_metadata_payload(&row.payload)?; + if &record.key() != key { + return Err(StoreError::LogicalKeyMismatch); + } + Ok(Some(record)) } /// Read every local metadata record in stable insertion order. @@ -662,17 +651,7 @@ impl Store { /// Delete one local metadata record. Returns whether it existed. pub fn delete_local_metadata(&self, key: &LocalMetadataKey) -> Result { - let rowid = self - .local_metadata_rows()? - .into_iter() - .find_map(|(rowid, record)| (&record.key() == key).then_some(rowid)); - let Some(rowid) = rowid else { - return Ok(false); - }; - Ok(self.conn.execute( - "DELETE FROM local_metadata WHERE rowid_ = ?1", - params![rowid], - )? == 1) + self.delete_equality::(&local_metadata_key(key)?) } /// Create a folder with a cryptographically random stable id. @@ -724,24 +703,16 @@ impl Store { let mut updates = Vec::new(); if compact { - for (position, (rowid, mut record)) in existing.into_iter().enumerate() { + for (position, (_, mut record)) in existing.into_iter().enumerate() { record.order = u32::try_from(position).map_err(|_| StoreError::FolderLimit)?; - let sealed = self.seal_local_metadata(&LocalMetadataRecord::Folder(record), rng)?; - updates.push((rowid, sealed)); + updates.push(LocalMetadataRecord::Folder(record)); } } - let sealed = self.seal_local_metadata(&LocalMetadataRecord::Folder(folder.clone()), rng)?; let tx = self.conn.unchecked_transaction()?; - for (rowid, sealed) in updates { - tx.execute( - "UPDATE local_metadata SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; - } - tx.execute( - "INSERT INTO local_metadata (blob) VALUES (?1)", - params![sealed], - )?; + for record in updates { + self.put_local_metadata_on(&tx, &record, rng)?; + } + self.put_local_metadata_on(&tx, &LocalMetadataRecord::Folder(folder.clone()), rng)?; tx.commit()?; Ok(folder) } @@ -781,21 +752,17 @@ impl Store { rng: &mut impl CryptoRngCore, ) -> Result { validate_new_folder(name)?; - let (rowid, mut folder) = self + let mut folder = self .local_metadata_rows()? .into_iter() - .find_map(|(rowid, record)| match record { - LocalMetadataRecord::Folder(folder) if folder.id == *id => Some((rowid, folder)), + .find_map(|(_, record)| match record { + LocalMetadataRecord::Folder(folder) if folder.id == *id => Some(folder), _ => None, }) .ok_or(StoreError::UnknownFolder)?; folder.name = name.to_owned(); - let sealed = self.seal_local_metadata(&LocalMetadataRecord::Folder(folder.clone()), rng)?; let tx = self.conn.unchecked_transaction()?; - tx.execute( - "UPDATE local_metadata SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; + self.put_local_metadata_on(&tx, &LocalMetadataRecord::Folder(folder.clone()), rng)?; tx.commit()?; Ok(folder) } @@ -827,22 +794,17 @@ impl Store { let mut reordered = Vec::with_capacity(ordered.len()); let mut updates = Vec::with_capacity(ordered.len()); for (position, id) in ordered.iter().enumerate() { - let (rowid, mut folder) = existing + let (_, mut folder) = existing .get(id) .cloned() .ok_or(StoreError::InvalidFolderOrder)?; folder.order = u32::try_from(position).map_err(|_| StoreError::InvalidFolderOrder)?; - let sealed = - self.seal_local_metadata(&LocalMetadataRecord::Folder(folder.clone()), rng)?; - updates.push((rowid, sealed)); + updates.push(LocalMetadataRecord::Folder(folder.clone())); reordered.push(folder); } let tx = self.conn.unchecked_transaction()?; - for (rowid, sealed) in updates { - tx.execute( - "UPDATE local_metadata SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; + for record in updates { + self.put_local_metadata_on(&tx, &record, rng)?; } tx.commit()?; Ok(reordered) @@ -865,42 +827,36 @@ impl Store { /// Atomically delete a folder and every assignment that points to it. pub fn delete_folder(&self, id: &[u8; 16]) -> Result { let rows = self.local_metadata_rows()?; - let mut folder_row = None; - let mut assignment_rows = Vec::new(); - let mut icon_row = None; - for (rowid, record) in rows { + let mut folder_found = false; + let mut assignment_keys = Vec::new(); + let mut icon_key = None; + for (_, record) in rows { match record { - LocalMetadataRecord::Folder(folder) if folder.id == *id => folder_row = Some(rowid), + LocalMetadataRecord::Folder(folder) if folder.id == *id => folder_found = true, LocalMetadataRecord::FolderAssignment(assignment) if assignment.folder == *id => { - assignment_rows.push(rowid) + assignment_keys + .push(LocalMetadataKey::FolderAssignment(assignment.conversation)) } LocalMetadataRecord::CustomIcon(icon) if icon.target == CustomIconTarget::Folder(*id) => { - icon_row = Some(rowid) + icon_key = Some(LocalMetadataKey::CustomIcon(icon.target)) } _ => {} } } - let folder_row = folder_row.ok_or(StoreError::UnknownFolder)?; - let count = assignment_rows.len(); + if !folder_found { + return Err(StoreError::UnknownFolder); + } + let count = assignment_keys.len(); let tx = self.conn.unchecked_transaction()?; - for rowid in assignment_rows { - tx.execute( - "DELETE FROM local_metadata WHERE rowid_ = ?1", - params![rowid], - )?; - } - if let Some(rowid) = icon_row { - tx.execute( - "DELETE FROM local_metadata WHERE rowid_ = ?1", - params![rowid], - )?; - } - tx.execute( - "DELETE FROM local_metadata WHERE rowid_ = ?1", - params![folder_row], - )?; + for key in assignment_keys { + self.delete_local_metadata_on(&tx, &key)?; + } + if let Some(key) = icon_key { + self.delete_local_metadata_on(&tx, &key)?; + } + self.delete_local_metadata_on(&tx, &LocalMetadataKey::Folder(*id))?; tx.commit()?; Ok(count) } @@ -948,19 +904,8 @@ impl Store { conversation: conversation.clone(), folder: *folder, }); - let sealed = self.seal_local_metadata(&record, rng)?; let tx = self.conn.unchecked_transaction()?; - if let Some((rowid, _)) = existing { - tx.execute( - "UPDATE local_metadata SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; - } else { - tx.execute( - "INSERT INTO local_metadata (blob) VALUES (?1)", - params![sealed], - )?; - } + self.put_local_metadata_on(&tx, &record, rng)?; tx.commit()?; Ok(true) } @@ -1115,27 +1060,7 @@ impl Store { } fn remove_folder_assignment(&self, conversation: &ConversationId) -> Result { - let rowid = - self.local_metadata_rows()? - .into_iter() - .find_map(|(rowid, record)| match record { - LocalMetadataRecord::FolderAssignment(assignment) - if assignment.conversation == *conversation => - { - Some(rowid) - } - _ => None, - }); - let Some(rowid) = rowid else { - return Ok(false); - }; - let tx = self.conn.unchecked_transaction()?; - tx.execute( - "DELETE FROM local_metadata WHERE rowid_ = ?1", - params![rowid], - )?; - tx.commit()?; - Ok(true) + self.delete_local_metadata(&LocalMetadataKey::FolderAssignment(conversation.clone())) } /// Idempotently pin one exact available typed conversation. @@ -1180,28 +1105,20 @@ impl Store { let mut updates = Vec::new(); if compact { - for (position, (rowid, mut pin)) in pins.into_iter().enumerate() { + for (position, (_, mut pin)) in pins.into_iter().enumerate() { pin.order = u32::try_from(position).map_err(|_| StoreError::PinLimit)?; - let sealed = self.seal_local_metadata(&LocalMetadataRecord::Pin(pin), rng)?; - updates.push((rowid, sealed)); + updates.push(LocalMetadataRecord::Pin(pin)); } } let record = PinRecord { conversation: conversation.clone(), order, }; - let sealed = self.seal_local_metadata(&LocalMetadataRecord::Pin(record), rng)?; let tx = self.conn.unchecked_transaction()?; - for (rowid, sealed) in updates { - tx.execute( - "UPDATE local_metadata SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; - } - tx.execute( - "INSERT INTO local_metadata (blob) VALUES (?1)", - params![sealed], - )?; + for record in updates { + self.put_local_metadata_on(&tx, &record, rng)?; + } + self.put_local_metadata_on(&tx, &LocalMetadataRecord::Pin(record), rng)?; tx.commit()?; Ok(true) } @@ -1275,20 +1192,16 @@ impl Store { } let mut updates = Vec::with_capacity(ordered.len()); for (position, conversation) in ordered.iter().enumerate() { - let (rowid, mut pin) = current + let (_, mut pin) = current .get(conversation) .cloned() .ok_or(StoreError::InvalidPinOrder)?; pin.order = u32::try_from(position).map_err(|_| StoreError::PinLimit)?; - let sealed = self.seal_local_metadata(&LocalMetadataRecord::Pin(pin), rng)?; - updates.push((rowid, sealed)); + updates.push(LocalMetadataRecord::Pin(pin)); } let tx = self.conn.unchecked_transaction()?; - for (rowid, sealed) in updates { - tx.execute( - "UPDATE local_metadata SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; + for record in updates { + self.put_local_metadata_on(&tx, &record, rng)?; } tx.commit()?; self.pins() @@ -1354,25 +1267,7 @@ impl Store { } fn remove_pin(&self, conversation: &ConversationId) -> Result { - let rowid = - self.local_metadata_rows()? - .into_iter() - .find_map(|(rowid, record)| match record { - LocalMetadataRecord::Pin(pin) if pin.conversation == *conversation => { - Some(rowid) - } - _ => None, - }); - let Some(rowid) = rowid else { - return Ok(false); - }; - let tx = self.conn.unchecked_transaction()?; - tx.execute( - "DELETE FROM local_metadata WHERE rowid_ = ?1", - params![rowid], - )?; - tx.commit()?; - Ok(true) + self.delete_local_metadata(&LocalMetadataKey::Pin(conversation.clone())) } /// Create a label with a cryptographically random stable id. @@ -1409,13 +1304,8 @@ impl Store { name: name.to_owned(), color: color.to_owned(), }; - let sealed = - self.seal_local_metadata(&LocalMetadataRecord::Label(label.clone()), rng)?; let tx = self.conn.unchecked_transaction()?; - tx.execute( - "INSERT INTO local_metadata (blob) VALUES (?1)", - params![sealed], - )?; + self.put_local_metadata_on(&tx, &LocalMetadataRecord::Label(label.clone()), rng)?; tx.commit()?; return Ok(label); } @@ -1455,25 +1345,16 @@ impl Store { rng: &mut impl CryptoRngCore, ) -> Result { validate_new_label(name, color)?; - let rowid = self - .local_metadata_rows()? - .into_iter() - .find_map(|(rowid, record)| match record { - LocalMetadataRecord::Label(label) if label.id == *id => Some(rowid), - _ => None, - }) - .ok_or(StoreError::UnknownLabel)?; + if self.label(id)?.is_none() { + return Err(StoreError::UnknownLabel); + } let label = LabelRecord { id: *id, name: name.to_owned(), color: color.to_owned(), }; - let sealed = self.seal_local_metadata(&LocalMetadataRecord::Label(label.clone()), rng)?; let tx = self.conn.unchecked_transaction()?; - tx.execute( - "UPDATE local_metadata SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; + self.put_local_metadata_on(&tx, &LocalMetadataRecord::Label(label.clone()), rng)?; tx.commit()?; Ok(label) } @@ -1498,30 +1379,29 @@ impl Store { /// complete cascade back, so restart cannot expose half-deleted state. pub fn delete_label(&self, id: &[u8; 16]) -> Result { let rows = self.local_metadata_rows()?; - let mut label_row = None; - let mut assignment_rows = Vec::new(); - for (rowid, record) in rows { + let mut label_found = false; + let mut assignment_keys = Vec::new(); + for (_, record) in rows { match record { - LocalMetadataRecord::Label(label) if label.id == *id => label_row = Some(rowid), + LocalMetadataRecord::Label(label) if label.id == *id => label_found = true, LocalMetadataRecord::LabelAssignment(assignment) if assignment.label == *id => { - assignment_rows.push(rowid); + assignment_keys.push(LocalMetadataKey::LabelAssignment( + assignment.label, + assignment.conversation, + )); } _ => {} } } - let label_row = label_row.ok_or(StoreError::UnknownLabel)?; - let assignment_count = assignment_rows.len(); + if !label_found { + return Err(StoreError::UnknownLabel); + } + let assignment_count = assignment_keys.len(); let tx = self.conn.unchecked_transaction()?; - for rowid in assignment_rows { - tx.execute( - "DELETE FROM local_metadata WHERE rowid_ = ?1", - params![rowid], - )?; - } - tx.execute( - "DELETE FROM local_metadata WHERE rowid_ = ?1", - params![label_row], - )?; + for key in assignment_keys { + self.delete_local_metadata_on(&tx, &key)?; + } + self.delete_local_metadata_on(&tx, &LocalMetadataKey::Label(*id))?; tx.commit()?; Ok(assignment_count) } @@ -1571,12 +1451,8 @@ impl Store { label: *label, conversation: conversation.clone(), }); - let sealed = self.seal_local_metadata(&assignment, rng)?; let tx = self.conn.unchecked_transaction()?; - tx.execute( - "INSERT INTO local_metadata (blob) VALUES (?1)", - params![sealed], - )?; + self.put_local_metadata_on(&tx, &assignment, rng)?; tx.commit()?; Ok(true) } @@ -1586,28 +1462,10 @@ impl Store { /// Returns `true` when a row was deleted and `false` for the honest absent /// no-op. Target availability is deliberately not required for cleanup. pub fn unassign_label(&self, label: &[u8; 16], conversation: &ConversationId) -> Result { - let rowid = - self.local_metadata_rows()? - .into_iter() - .find_map(|(rowid, record)| match record { - LocalMetadataRecord::LabelAssignment(assignment) - if assignment.label == *label - && assignment.conversation == *conversation => - { - Some(rowid) - } - _ => None, - }); - let Some(rowid) = rowid else { - return Ok(false); - }; - let tx = self.conn.unchecked_transaction()?; - tx.execute( - "DELETE FROM local_metadata WHERE rowid_ = ?1", - params![rowid], - )?; - tx.commit()?; - Ok(true) + self.delete_local_metadata(&LocalMetadataKey::LabelAssignment( + *label, + conversation.clone(), + )) } /// Active conversation membership for one label in durable insertion order. @@ -1839,42 +1697,61 @@ impl Store { Ok(activity) } - fn seal_local_metadata( + fn put_local_metadata_on( &self, + conn: &Connection, record: &LocalMetadataRecord, rng: &mut impl CryptoRngCore, - ) -> Result> { + ) -> Result<()> { + record.validate()?; let encoded = postcard::to_allocvec(record).map_err(|_| StoreError::Serialization)?; let mut versioned = Vec::with_capacity(RECORD_MAGIC_V1.len() + encoded.len()); versioned.extend_from_slice(RECORD_MAGIC_V1); versioned.extend_from_slice(&encoded); - Ok(self.k_local_metadata.seal(RECORD_AD, &versioned, rng)) + let versioned = Zeroizing::new(versioned); + self.put_equality_on::( + conn, + &local_metadata_key(&record.key())?, + &versioned, + store_v2::IndexKeys::none(), + rng, + ) + } + + fn delete_local_metadata_on(&self, conn: &Connection, key: &LocalMetadataKey) -> Result { + self.delete_equality_on::(conn, &local_metadata_key(key)?) } fn local_metadata_rows(&self) -> Result> { - let mut stmt = self - .conn - .prepare("SELECT rowid_, blob FROM local_metadata ORDER BY rowid_")?; - let rows = stmt.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - })?; let mut records = Vec::new(); - for row in rows { - let (rowid, sealed) = row?; - let plain = self.k_local_metadata.open(RECORD_AD, &sealed)?; - let encoded = plain - .strip_prefix(RECORD_MAGIC_V1) - .ok_or(StoreError::Serialization)?; - let (record, remainder): (LocalMetadataRecord, &[u8]) = - postcard::take_from_bytes(encoded).map_err(|_| StoreError::Serialization)?; - if !remainder.is_empty() { - return Err(StoreError::Serialization); - } - record.validate()?; - records.push((rowid, record)); + for row in self.rows::()? { + let record = decode_local_metadata_payload(&row.payload)?; + row.verify_key(&local_metadata_key(&record.key())?)?; + records.push((row.rowid, record)); } Ok(records) } + + pub(crate) fn validate_local_metadata_logical_rows(&self) -> Result<()> { + self.validate_rows::(|row| { + let record = decode_local_metadata_payload(&row.payload)?; + row.verify_key(&local_metadata_key(&record.key())?)?; + row.verify_indexes(&store_v2::IndexKeys::none()) + }) + } +} + +fn local_metadata_key(key: &LocalMetadataKey) -> Result { + store_v2::MetadataKey::new(postcard::to_allocvec(key).map_err(|_| StoreError::Serialization)?) +} + +fn decode_local_metadata_payload(payload: &[u8]) -> Result { + let encoded = payload + .strip_prefix(RECORD_MAGIC_V1) + .ok_or(StoreError::Serialization)?; + let record: LocalMetadataRecord = decode_exact(encoded)?; + record.validate()?; + Ok(record) } fn compare_pins( @@ -2020,12 +1897,7 @@ mod label_tests { let tx = store.conn.unchecked_transaction().unwrap(); for record in records { record.validate().unwrap(); - let sealed = store.seal_local_metadata(&record, rng).unwrap(); - tx.execute( - "INSERT INTO local_metadata (blob) VALUES (?1)", - params![sealed], - ) - .unwrap(); + store.put_local_metadata_on(&tx, &record, rng).unwrap(); } tx.commit().unwrap(); } @@ -2403,14 +2275,16 @@ mod label_tests { let raw = Connection::open(&db).unwrap(); raw.execute_batch( - "CREATE TRIGGER fail_folder_create BEFORE INSERT ON local_metadata BEGIN SELECT RAISE(FAIL, 'injected folder create'); END;", + "CREATE TRIGGER fail_folder_create BEFORE INSERT ON store_records + BEGIN SELECT RAISE(FAIL, 'injected folder create'); END;", ) .unwrap(); drop(raw); - assert!(matches!( - store.create_folder("create fails", &mut rng), - Err(StoreError::Db(_)) - )); + let result = store.create_folder("create fails", &mut rng); + assert!( + matches!(result, Err(StoreError::Db(_))), + "unexpected result: {result:?}" + ); let raw = Connection::open(&db).unwrap(); raw.execute_batch("DROP TRIGGER fail_folder_create") .unwrap(); @@ -2426,13 +2300,16 @@ mod label_tests { let raw = Connection::open(&db).unwrap(); let first_row: i64 = raw .query_row( - "SELECT rowid_ FROM local_metadata ORDER BY rowid_ LIMIT 1", + "SELECT rowid_ FROM store_records + WHERE table_domain = 18 ORDER BY rowid_ LIMIT 1", [], |row| row.get(0), ) .unwrap(); raw.execute_batch( - "CREATE TRIGGER fail_folder_rename BEFORE UPDATE ON local_metadata BEGIN SELECT RAISE(FAIL, 'injected folder rename'); END;", + "CREATE TRIGGER fail_folder_rename BEFORE UPDATE ON store_records + WHEN NEW.table_domain = 18 + BEGIN SELECT RAISE(FAIL, 'injected folder rename'); END;", ) .unwrap(); drop(raw); @@ -2444,7 +2321,9 @@ mod label_tests { raw.execute_batch("DROP TRIGGER fail_folder_rename") .unwrap(); raw.execute_batch(&format!( - "CREATE TRIGGER fail_folder_reorder BEFORE UPDATE ON local_metadata WHEN OLD.rowid_ = {first_row} BEGIN SELECT RAISE(FAIL, 'injected folder reorder'); END;" + "CREATE TRIGGER fail_folder_reorder BEFORE UPDATE ON store_records + WHEN OLD.table_domain = 18 AND OLD.rowid_ = {first_row} + BEGIN SELECT RAISE(FAIL, 'injected folder reorder'); END;" )) .unwrap(); drop(raw); @@ -2456,7 +2335,9 @@ mod label_tests { raw.execute_batch("DROP TRIGGER fail_folder_reorder") .unwrap(); raw.execute_batch(&format!( - "CREATE TRIGGER fail_folder_cascade BEFORE DELETE ON local_metadata WHEN OLD.rowid_ = {first_row} BEGIN SELECT RAISE(FAIL, 'injected folder cascade'); END;" + "CREATE TRIGGER fail_folder_cascade BEFORE DELETE ON store_records + WHEN OLD.table_domain = 18 AND OLD.rowid_ = {first_row} + BEGIN SELECT RAISE(FAIL, 'injected folder cascade'); END;" )) .unwrap(); drop(raw); @@ -2832,14 +2713,16 @@ mod label_tests { let raw = Connection::open(&db).unwrap(); raw.execute_batch( - "CREATE TRIGGER fail_create BEFORE INSERT ON local_metadata BEGIN SELECT RAISE(FAIL, 'injected create'); END;", + "CREATE TRIGGER fail_create BEFORE INSERT ON store_records + BEGIN SELECT RAISE(FAIL, 'injected create'); END;", ) .unwrap(); drop(raw); - assert!(matches!( - store.create_label("create fails", "red", &mut rng), - Err(StoreError::Db(_)) - )); + let result = store.create_label("create fails", "red", &mut rng); + assert!( + matches!(result, Err(StoreError::Db(_))), + "unexpected result: {result:?}" + ); let raw = Connection::open(&db).unwrap(); raw.execute_batch("DROP TRIGGER fail_create").unwrap(); drop(raw); @@ -2851,7 +2734,9 @@ mod label_tests { .unwrap(); let raw = Connection::open(&db).unwrap(); raw.execute_batch( - "CREATE TRIGGER fail_update BEFORE UPDATE ON local_metadata BEGIN SELECT RAISE(FAIL, 'injected update'); END;", + "CREATE TRIGGER fail_update BEFORE UPDATE ON store_records + WHEN NEW.table_domain = 18 + BEGIN SELECT RAISE(FAIL, 'injected update'); END;", ) .unwrap(); drop(raw); @@ -2866,7 +2751,9 @@ mod label_tests { let raw = Connection::open(&db).unwrap(); raw.execute_batch( - "CREATE TRIGGER fail_cascade BEFORE DELETE ON local_metadata WHEN OLD.rowid_ = 1 BEGIN SELECT RAISE(FAIL, 'injected cascade'); END;", + "CREATE TRIGGER fail_cascade BEFORE DELETE ON store_records + WHEN OLD.table_domain = 18 + BEGIN SELECT RAISE(FAIL, 'injected cascade'); END;", ) .unwrap(); drop(raw); @@ -2895,13 +2782,16 @@ mod label_tests { drop(store); let raw = Connection::open(&db).unwrap(); raw.execute( - "UPDATE local_metadata SET blob = zeroblob(length(blob))", + "UPDATE store_records SET blob = zeroblob(length(blob)) + WHERE table_domain = 18", [], ) .unwrap(); drop(raw); - let reopened = Store::open(&db, b"pass").unwrap(); - assert!(matches!(reopened.labels(), Err(StoreError::Crypto(_)))); + assert!(matches!( + Store::open(&db, b"pass"), + Err(StoreError::Crypto(_)) + )); } #[test] @@ -3063,7 +2953,9 @@ mod label_tests { let (append, mut rng) = store_at(&append_db, 0xb111); let raw = Connection::open(&append_db).unwrap(); raw.execute_batch( - "CREATE TRIGGER fail_pin_append BEFORE INSERT ON local_metadata BEGIN SELECT RAISE(FAIL, 'injected pin append'); END;", + "CREATE TRIGGER fail_pin_append BEFORE INSERT ON store_records + WHEN NEW.table_domain = 18 + BEGIN SELECT RAISE(FAIL, 'injected pin append'); END;", ) .unwrap(); assert!(append @@ -3089,7 +2981,9 @@ mod label_tests { ); let raw = Connection::open(&compact_db).unwrap(); raw.execute_batch( - "CREATE TRIGGER fail_pin_compact_insert BEFORE INSERT ON local_metadata BEGIN SELECT RAISE(FAIL, 'injected pin compact insert'); END;", + "CREATE TRIGGER fail_pin_compact_insert BEFORE INSERT ON store_records + WHEN NEW.table_domain = 18 + BEGIN SELECT RAISE(FAIL, 'injected pin compact insert'); END;", ) .unwrap(); assert!(compact @@ -3123,7 +3017,9 @@ mod label_tests { let before = mutations.pins().unwrap(); let raw = Connection::open(&mutation_db).unwrap(); raw.execute_batch( - "CREATE TRIGGER fail_pin_reorder BEFORE UPDATE ON local_metadata BEGIN SELECT RAISE(FAIL, 'injected pin reorder'); END;", + "CREATE TRIGGER fail_pin_reorder BEFORE UPDATE ON store_records + WHEN NEW.table_domain = 18 + BEGIN SELECT RAISE(FAIL, 'injected pin reorder'); END;", ) .unwrap(); assert!(mutations @@ -3135,7 +3031,9 @@ mod label_tests { assert_eq!(mutations.pins().unwrap(), before); raw.execute_batch("DROP TRIGGER fail_pin_reorder").unwrap(); raw.execute_batch( - "CREATE TRIGGER fail_pin_delete BEFORE DELETE ON local_metadata BEGIN SELECT RAISE(FAIL, 'injected pin delete'); END;", + "CREATE TRIGGER fail_pin_delete BEFORE DELETE ON store_records + WHEN OLD.table_domain = 18 + BEGIN SELECT RAISE(FAIL, 'injected pin delete'); END;", ) .unwrap(); assert!(mutations diff --git a/crates/kult-store/src/maintenance.rs b/crates/kult-store/src/maintenance.rs new file mode 100644 index 0000000..b2acd39 --- /dev/null +++ b/crates/kult-store/src/maintenance.rs @@ -0,0 +1,226 @@ +//! Bounded local remnant-reduction controls. + +use std::ffi::OsString; +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::{store_v2, Result, Store, StoreError}; + +/// Largest WAL file this API will synchronously checkpoint (256 MiB). +pub const MAX_MAINTENANCE_WAL_BYTES: u64 = 256 * 1024 * 1024; +/// Largest incremental-vacuum request accepted in one call. +pub const MAX_MAINTENANCE_VACUUM_PAGES: u32 = 4_096; + +/// Per-call bounds for local SQLite remnant reduction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StorageMaintenanceOptions { + /// Checkpoint and truncate the WAL only when its current file size is no + /// larger than this bound. + pub max_wal_checkpoint_bytes: u64, + /// Maximum freelist pages SQLite may reclaim during this call. + pub max_incremental_vacuum_pages: u32, +} + +impl Default for StorageMaintenanceOptions { + fn default() -> Self { + Self { + max_wal_checkpoint_bytes: 64 * 1024 * 1024, + max_incremental_vacuum_pages: 1_024, + } + } +} + +/// Observable result of one bounded maintenance call. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StorageMaintenanceReport { + /// Whether this SQLite build reports full secure-delete mode enabled. + pub secure_delete_enabled: bool, + /// Reclaimable database pages before incremental vacuum. + pub freelist_pages_before: u64, + /// Reclaimable database pages after incremental vacuum. + pub freelist_pages_after: u64, + /// WAL bytes observed before maintenance. + pub wal_bytes_before: u64, + /// WAL bytes observed after maintenance. + pub wal_bytes_after: u64, + /// Frames present when a bounded checkpoint was attempted. + pub wal_frames_before_checkpoint: u64, + /// Frames copied back to the database by that checkpoint. + pub wal_frames_checkpointed: u64, + /// Whether WAL truncation was skipped because the file exceeded the + /// caller's bound, or could not complete because another reader was active. + pub wal_checkpoint_deferred: bool, + /// Always false: SQLite and filesystem maintenance cannot prove forensic + /// erasure on snapshots, flash media, remapped blocks, or prior copies. + pub forensic_erasure_guaranteed: bool, +} + +impl Store { + /// Reduce deleted-row and WAL remnants within explicit per-call bounds. + /// + /// Logical deletion must happen before this call. This operation enables + /// full SQLite secure-delete mode, performs a bounded incremental vacuum, + /// and truncates only a WAL whose observed file size fits the supplied + /// checkpoint bound. It cannot establish forensic erasure. + pub fn maintain_deleted_storage( + &self, + options: StorageMaintenanceOptions, + ) -> Result { + if options.max_wal_checkpoint_bytes == 0 + || options.max_wal_checkpoint_bytes > MAX_MAINTENANCE_WAL_BYTES + || options.max_incremental_vacuum_pages > MAX_MAINTENANCE_VACUUM_PAGES + { + return Err(StoreError::MaintenanceBounds); + } + + let wal = sidecar_path(&self.path, "-wal"); + let wal_bytes_before = file_len(&wal)?; + self.conn.pragma_update(None, "secure_delete", true)?; + let secure_delete: i64 = self + .conn + .pragma_query_value(None, "secure_delete", |row| row.get(0))?; + let page_size: u64 = self + .conn + .pragma_query_value(None, "page_size", |row| row.get(0))?; + let checkpoint_pages = options + .max_wal_checkpoint_bytes + .checked_div(page_size.max(1)) + .unwrap_or(1) + .max(1); + self.conn.pragma_update( + None, + "wal_autocheckpoint", + u32::try_from(checkpoint_pages).unwrap_or(u32::MAX), + )?; + self.conn.pragma_update( + None, + "journal_size_limit", + i64::try_from(options.max_wal_checkpoint_bytes) + .map_err(|_| StoreError::MaintenanceBounds)?, + )?; + + let freelist_pages_before = pragma_u64(&self.conn, "freelist_count")?; + if options.max_incremental_vacuum_pages != 0 { + self.conn.execute_batch(&format!( + "PRAGMA incremental_vacuum({});", + options.max_incremental_vacuum_pages + ))?; + } + let freelist_pages_after = pragma_u64(&self.conn, "freelist_count")?; + + let mut wal_frames_before_checkpoint = 0; + let mut wal_frames_checkpointed = 0; + let mut wal_checkpoint_deferred = wal_bytes_before > options.max_wal_checkpoint_bytes; + if !wal_checkpoint_deferred { + let (busy, frames, checkpointed): (u32, u64, u64) = + self.conn + .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + })?; + wal_frames_before_checkpoint = frames; + wal_frames_checkpointed = checkpointed; + wal_checkpoint_deferred = busy != 0; + } + store_v2::protect_sqlite_files(&self.path)?; + let wal_bytes_after = file_len(&wal)?; + + Ok(StorageMaintenanceReport { + secure_delete_enabled: secure_delete == 1, + freelist_pages_before, + freelist_pages_after, + wal_bytes_before, + wal_bytes_after, + wal_frames_before_checkpoint, + wal_frames_checkpointed, + wal_checkpoint_deferred, + forensic_erasure_guaranteed: false, + }) + } +} + +fn pragma_u64(conn: &rusqlite::Connection, pragma: &str) -> Result { + let value: i64 = conn.pragma_query_value(None, pragma, |row| row.get(0))?; + u64::try_from(value).map_err(|_| StoreError::Serialization) +} + +fn file_len(path: &Path) -> Result { + match fs::metadata(path) { + Ok(metadata) => Ok(metadata.len()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(0), + Err(error) => Err(error.into()), + } +} + +fn sidecar_path(path: &Path, suffix: &str) -> PathBuf { + let mut value: OsString = path.as_os_str().to_owned(); + value.push(suffix); + PathBuf::from(value) +} + +#[cfg(test)] +mod tests { + use kult_crypto::KdfProfile; + use rand::{rngs::StdRng, SeedableRng}; + + use super::*; + use crate::{DeliveryState, Direction, MessageRecord}; + + const TEST_KDF: KdfProfile = KdfProfile { + m_cost_kib: 8, + t_cost: 1, + p_cost: 1, + }; + + #[test] + fn maintenance_is_bounded_and_never_claims_forensic_erasure() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("maintenance.db"); + let mut rng = StdRng::seed_from_u64(0x5ecde1); + let store = Store::create(&path, b"pass", TEST_KDF, &mut rng).unwrap(); + for value in 0..64u8 { + let mut id = [0; 16]; + id[0] = value; + let record = MessageRecord { + id, + peer: [7; 32], + direction: Direction::Outbound, + state: DeliveryState::Queued, + timestamp: u64::from(value), + body: vec![value; 8 * 1024], + wire_id: None, + }; + store.put_message(&record, &mut rng).unwrap(); + assert!(store + .delete_message_record(&record.peer, record.direction, &record.id) + .unwrap()); + } + + let report = store + .maintain_deleted_storage(StorageMaintenanceOptions::default()) + .unwrap(); + assert!(report.secure_delete_enabled); + assert!(report.freelist_pages_after <= report.freelist_pages_before); + assert!(!report.forensic_erasure_guaranteed); + assert!( + report.wal_checkpoint_deferred + || report.wal_bytes_after + <= StorageMaintenanceOptions::default().max_wal_checkpoint_bytes + ); + } + + #[test] + fn maintenance_rejects_work_above_the_public_limits() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("maintenance-bounds.db"); + let mut rng = StdRng::seed_from_u64(0x5ecde2); + let store = Store::create(&path, b"pass", TEST_KDF, &mut rng).unwrap(); + let options = StorageMaintenanceOptions { + max_wal_checkpoint_bytes: MAX_MAINTENANCE_WAL_BYTES + 1, + max_incremental_vacuum_pages: 0, + }; + assert!(matches!( + store.maintain_deleted_storage(options), + Err(StoreError::MaintenanceBounds) + )); + } +} diff --git a/crates/kult-store/src/media.rs b/crates/kult-store/src/media.rs index 7bf3a62..3e76cb0 100644 --- a/crates/kult-store/src/media.rs +++ b/crates/kult-store/src/media.rs @@ -2,13 +2,15 @@ use std::collections::HashSet; use std::ffi::OsString; -use std::fs::{self, File, OpenOptions}; +#[cfg(unix)] +use std::fs::File; +use std::fs::{self, OpenOptions}; use std::io::Write; use std::path::{Path, PathBuf}; use rand_core::CryptoRngCore; -use rusqlite::{params, OptionalExtension}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use zeroize::Zeroizing; use kult_crypto::bulk_hash; use kult_protocol::{ @@ -17,11 +19,9 @@ use kult_protocol::{ MAX_PRIMARY_OBJECT_LEN, }; -use crate::{Result, Store, StoreError}; +use crate::{store_v2, Result, Store, StoreError}; const MEDIA_RECORD_VERSION: u8 = 1; -const MEDIA_TRANSFER_AD: &[u8] = b"media-transfer"; -const MEDIA_OBJECT_AD: &[u8] = b"media-object"; const MEDIA_CHUNK_AD: &[u8] = b"KAT-store-chunk-v1"; const MAX_ENTITLEMENTS: usize = 64; const LOCAL_SEAL_OVERHEAD: u64 = 24 + 16; @@ -260,10 +260,12 @@ impl Store { rng: &mut impl CryptoRngCore, ) -> Result<()> { validate_transfer(record)?; - let sealed = seal_record(&self.k_media, MEDIA_TRANSFER_AD, record, rng)?; - self.conn.execute( - "INSERT OR REPLACE INTO media_transfers (id, blob) VALUES (?1, ?2)", - params![record.local_id.as_slice(), sealed], + let encoded = encode_record(record)?; + self.put_equality::( + &store_v2::LocalIdKey::new(record.local_id), + &encoded, + store_v2::IndexKeys::none(), + rng, )?; Ok(()) } @@ -273,12 +275,12 @@ impl Store { &self, local_id: &[u8; 16], ) -> Result>> { - self.get_media_record("media_transfers", MEDIA_TRANSFER_AD, local_id) + self.get_media_record::(local_id) } /// Read every transfer metadata row. pub fn media_transfers(&self) -> Result>> { - self.media_records("media_transfers", MEDIA_TRANSFER_AD) + self.media_records::() } /// Transition transfer-level lifecycle state and authenticated progress @@ -298,15 +300,10 @@ impl Store { /// Delete one transfer row after its object rows have been removed. pub fn delete_media_transfer(&self, local_id: &[u8; 16]) -> Result<()> { - if self.media_objects()?.into_iter().any( - |record| matches!(record, MediaRecord::Available(object) if object.transfer_id == *local_id), - ) { + if !self.media_objects_for_transfer(local_id)?.is_empty() { return Err(StoreError::MediaState); } - self.conn.execute( - "DELETE FROM media_transfers WHERE id = ?1", - params![local_id.as_slice()], - )?; + self.delete_equality::(&store_v2::LocalIdKey::new(*local_id))?; Ok(()) } @@ -335,10 +332,12 @@ impl Store { if record.state.active() { self.enforce_active_limits(&transfer, Some(record.local_id))?; } - let sealed = seal_record(&self.k_media, MEDIA_OBJECT_AD, record, rng)?; - self.conn.execute( - "INSERT OR REPLACE INTO media_objects (id, blob) VALUES (?1, ?2)", - params![record.local_id.as_slice(), sealed], + let encoded = encode_record(record)?; + self.put_equality::( + &store_v2::LocalIdKey::new(record.local_id), + &encoded, + store_v2::IndexKeys::media_object(&store_v2::LocalIdKey::new(record.transfer_id)), + rng, )?; Ok(()) } @@ -348,12 +347,12 @@ impl Store { &self, local_id: &[u8; 16], ) -> Result>> { - self.get_media_record("media_objects", MEDIA_OBJECT_AD, local_id) + self.get_media_record::(local_id) } /// Read every object metadata row. pub fn media_objects(&self) -> Result>> { - self.media_records("media_objects", MEDIA_OBJECT_AD) + self.media_records::() } /// Read supported object rows belonging to one transfer, preserving @@ -362,16 +361,20 @@ impl Store { &self, transfer_id: &[u8; 16], ) -> Result> { - let mut objects: Vec<_> = self - .media_objects()? - .into_iter() - .filter_map(|record| match record { - MediaRecord::Available(object) if object.transfer_id == *transfer_id => { - Some(object) + let mut objects = Vec::new(); + for row in self.rows_by_index::( + &store_v2::LocalIdKey::new(*transfer_id), + )? { + let local_id = *store_v2::LocalIdKey::decode(&row.logical_key)?.value(); + if let MediaRecord::Available(object) = + self.decode_media_record::(local_id, &row.payload)? + { + if object.transfer_id != *transfer_id { + return Err(StoreError::LogicalKeyMismatch); } - _ => None, - }) - .collect(); + objects.push(object); + } + } objects.sort_by_key(|object| object.role); Ok(objects) } @@ -492,13 +495,7 @@ impl Store { if !matches!(object.state, MediaTransferState::Complete) { object.state = MediaTransferState::Transferring; } - let sealed = seal_record(&self.k_media, MEDIA_OBJECT_AD, &object, rng)?; - let tx = self.conn.transaction()?; - tx.execute( - "UPDATE media_objects SET blob = ?2 WHERE id = ?1", - params![object.local_id.as_slice(), sealed], - )?; - tx.commit()?; + self.persist_object(&object, rng)?; Ok(address) } @@ -552,12 +549,7 @@ impl Store { /// Delete one object row and garbage-collect chunk files no longer referenced. pub fn delete_media_object(&mut self, local_id: &[u8; 16]) -> Result<()> { - let tx = self.conn.transaction()?; - tx.execute( - "DELETE FROM media_objects WHERE id = ?1", - params![local_id.as_slice()], - )?; - tx.commit()?; + self.delete_equality::(&store_v2::LocalIdKey::new(*local_id))?; self.garbage_collect_media_files(false).map(|_| ()) } @@ -626,48 +618,37 @@ impl Store { Ok(report) } - fn get_media_record( - &self, - table: &str, - ad: &[u8], - local_id: &[u8; 16], - ) -> Result>> { - let sql = format!("SELECT blob FROM {table} WHERE id = ?1"); - let sealed: Option> = self - .conn - .query_row(&sql, params![local_id.as_slice()], |row| row.get(0)) - .optional()?; - sealed - .map(|sealed| self.decode_media_record(ad, *local_id, &sealed)) - .transpose() - } - - fn media_records( - &self, - table: &str, - ad: &[u8], - ) -> Result>> { - let sql = format!("SELECT id, blob FROM {table} ORDER BY rowid"); - let mut stmt = self.conn.prepare(&sql)?; - let rows = stmt.query_map([], |row| { - Ok((row.get::<_, Vec>(0)?, row.get::<_, Vec>(1)?)) - })?; + fn get_media_record(&self, local_id: &[u8; 16]) -> Result>> + where + T: DeserializeOwned, + R: store_v2::TableSpec, + { + let key = store_v2::LocalIdKey::new(*local_id); + let Some(row) = self.get_equality::(&key)? else { + return Ok(None); + }; + row.verify_key(&key)?; + self.decode_media_record(*local_id, &row.payload).map(Some) + } + + fn media_records(&self) -> Result>> + where + T: DeserializeOwned, + R: store_v2::TableSpec, + { let mut records = Vec::new(); - for row in rows { - let (local_id, sealed) = row?; - let local_id: [u8; 16] = local_id.try_into().map_err(|_| StoreError::Serialization)?; - records.push(self.decode_media_record(ad, local_id, &sealed)?); + for row in self.rows::()? { + let local_id = *store_v2::LocalIdKey::decode(&row.logical_key)?.value(); + records.push(self.decode_media_record(local_id, &row.payload)?); } Ok(records) } fn decode_media_record( &self, - ad: &[u8], local_id: [u8; 16], - sealed: &[u8], + plain: &[u8], ) -> Result> { - let plain = self.k_media.open(ad, sealed)?; let Some((&version, encoded)) = plain.split_first() else { return Err(StoreError::Serialization); }; @@ -702,10 +683,12 @@ impl Store { rng: &mut impl CryptoRngCore, ) -> Result<()> { validate_object(object)?; - let sealed = seal_record(&self.k_media, MEDIA_OBJECT_AD, object, rng)?; - self.conn.execute( - "UPDATE media_objects SET blob = ?2 WHERE id = ?1", - params![object.local_id.as_slice(), sealed], + let encoded = encode_record(object)?; + self.put_equality::( + &store_v2::LocalIdKey::new(object.local_id), + &encoded, + store_v2::IndexKeys::media_object(&store_v2::LocalIdKey::new(object.transfer_id)), + rng, )?; Ok(()) } @@ -772,19 +755,45 @@ impl Store { } Ok(removed) } + + pub(crate) fn validate_media_logical_rows(&self) -> Result<()> { + self.validate_rows::(|row| { + let local_id = *store_v2::LocalIdKey::decode(&row.logical_key)?.value(); + row.verify_indexes(&store_v2::IndexKeys::none())?; + if let MediaRecord::Available(record) = + self.decode_media_record::(local_id, &row.payload)? + { + validate_transfer(&record)?; + if record.local_id != local_id { + return Err(StoreError::LogicalKeyMismatch); + } + } + Ok(()) + })?; + self.validate_rows::(|row| { + let local_id = *store_v2::LocalIdKey::decode(&row.logical_key)?.value(); + if let MediaRecord::Available(record) = + self.decode_media_record::(local_id, &row.payload)? + { + validate_object(&record)?; + if record.local_id != local_id { + return Err(StoreError::LogicalKeyMismatch); + } + row.verify_indexes(&store_v2::IndexKeys::media_object( + &store_v2::LocalIdKey::new(record.transfer_id), + ))?; + } + Ok(()) + }) + } } -fn seal_record( - key: &kult_crypto::StorageKey, - ad: &[u8], - record: &T, - rng: &mut impl CryptoRngCore, -) -> Result> { +fn encode_record(record: &T) -> Result>> { let encoded = postcard::to_allocvec(record).map_err(|_| StoreError::Serialization)?; let mut versioned = Vec::with_capacity(1 + encoded.len()); versioned.push(MEDIA_RECORD_VERSION); versioned.extend_from_slice(&encoded); - Ok(key.seal(ad, &versioned, rng)) + Ok(Zeroizing::new(versioned)) } fn validate_transfer(record: &MediaTransferRecord) -> Result<()> { @@ -1000,14 +1009,12 @@ mod tests { let store = Store::create(&dir.path().join("versions.db"), b"pass", TEST_KDF, &mut rng).unwrap(); let local_id = [9u8; 16]; - let sealed = store - .k_media - .seal(MEDIA_OBJECT_AD, &[2, 0xff, 0xff], &mut rng); store - .conn - .execute( - "INSERT INTO media_objects (id, blob) VALUES (?1, ?2)", - params![local_id.as_slice(), sealed], + .put_equality::( + &store_v2::LocalIdKey::new(local_id), + &[2, 0xff, 0xff], + store_v2::IndexKeys::media_object(&store_v2::LocalIdKey::new([7; 16])), + &mut rng, ) .unwrap(); assert_eq!( diff --git a/crates/kult-store/src/migration.rs b/crates/kult-store/src/migration.rs new file mode 100644 index 0000000..1546747 --- /dev/null +++ b/crates/kult-store/src/migration.rs @@ -0,0 +1,1390 @@ +//! Released-schema migration into the ADR-0027 opaque store. + +use std::fs::{self, File, OpenOptions}; +use std::path::{Path, PathBuf}; + +use rand_core::OsRng; +use rusqlite::types::Value; +use rusqlite::{params, Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; +use zeroize::Zeroizing; + +use kult_crypto::{derive_kek, Identity, KdfProfile, Session, StorageKey}; +use kult_protocol::{CapabilityControl, Envelope}; + +use crate::{ + store_lock_path, store_v2, ContactDeviceRecord, ContactRecord, DeviceStateRecord, + EphemeralRecord, GroupAuthorityRecord, GroupMessageRecord, GroupRecord, LocalMetadataRecord, + MediaObjectRecord, MediaTransferRecord, MessageDeviceDeliveryRecord, MessageRecord, + NoteMessageRecord, Result, ScheduledMessageRecord, Store, StoreError, +}; + +const SCHEMA_V0_1: &str = include_str!("../tests/fixtures/schema-v0.1.0.sql"); +const SCHEMA_V0_2: &str = include_str!("../tests/fixtures/schema-v0.2.0.sql"); +const SCHEMA_V0_3: &str = include_str!("../tests/fixtures/schema-v0.3.0.sql"); +const BATCH_ROWS: usize = 256; +const MAX_TOTAL_ROWS: u64 = 50_000_000; +const MAX_LEGACY_CIPHERTEXT: usize = store_v2::MAX_RECORD_BYTES + 128; +const WORKSPACE_RESERVE: u64 = 64 * 1024 * 1024; +const ESTIMATED_OPAQUE_ROW_OVERHEAD: u64 = 1_024; +const CHECKPOINT_TABLES: usize = 25; +const SOURCE_FINGERPRINT_DOMAIN: &[u8] = b"Komms-Store-Legacy-Fingerprint-v2"; +const WRAP_AD: &[u8] = b"KK-store-wrap-v1"; + +#[derive(Clone, Copy)] +struct TableSpec { + name: &'static str, + columns: &'static str, + max_rows: u64, + optional_receipt: bool, +} + +const TABLES: [TableSpec; CHECKPOINT_TABLES] = [ + TableSpec { + name: "identity", + columns: "blob", + max_rows: 1, + optional_receipt: false, + }, + TableSpec { + name: "sessions", + columns: "peer, blob", + max_rows: 100_000, + optional_receipt: false, + }, + TableSpec { + name: "capabilities", + columns: "peer, blob", + max_rows: 100_000, + optional_receipt: false, + }, + TableSpec { + name: "messages", + columns: "blob", + max_rows: 10_000_000, + optional_receipt: false, + }, + TableSpec { + name: "queue", + columns: "blob", + max_rows: 1_000_000, + optional_receipt: false, + }, + TableSpec { + name: "seen", + columns: "id", + max_rows: 10_000_000, + optional_receipt: false, + }, + TableSpec { + name: "receipt_replay", + columns: "id, blob", + max_rows: 1_000_000, + optional_receipt: true, + }, + TableSpec { + name: "contacts", + columns: "peer, blob", + max_rows: 100_000, + optional_receipt: false, + }, + TableSpec { + name: "prekeys", + columns: "blob", + max_rows: 1, + optional_receipt: false, + }, + TableSpec { + name: "pending", + columns: "blob", + max_rows: 100_000, + optional_receipt: false, + }, + TableSpec { + name: "resets", + columns: "peer", + max_rows: 100_000, + optional_receipt: false, + }, + TableSpec { + name: "groups", + columns: "gid, blob", + max_rows: 100_000, + optional_receipt: false, + }, + TableSpec { + name: "group_authority", + columns: "gid, blob", + max_rows: 100_000, + optional_receipt: false, + }, + TableSpec { + name: "group_chains", + columns: "gid, peer, blob", + max_rows: 1_000_000, + optional_receipt: false, + }, + TableSpec { + name: "group_msgs", + columns: "blob", + max_rows: 10_000_000, + optional_receipt: false, + }, + TableSpec { + name: "media_transfers", + columns: "id, blob", + max_rows: 1_000_000, + optional_receipt: false, + }, + TableSpec { + name: "media_objects", + columns: "id, blob", + max_rows: 2_000_000, + optional_receipt: false, + }, + TableSpec { + name: "local_metadata", + columns: "blob", + max_rows: 100_000, + optional_receipt: false, + }, + TableSpec { + name: "note_messages", + columns: "blob", + max_rows: 10_000_000, + optional_receipt: false, + }, + TableSpec { + name: "scheduled_messages", + columns: "blob", + max_rows: 1_000_000, + optional_receipt: false, + }, + TableSpec { + name: "ephemeral", + columns: "blob", + max_rows: 10_000_000, + optional_receipt: false, + }, + TableSpec { + name: "device_state", + columns: "blob", + max_rows: 1, + optional_receipt: false, + }, + TableSpec { + name: "device_sync", + columns: "blob", + max_rows: 100_000, + optional_receipt: false, + }, + TableSpec { + name: "contact_devices", + columns: "blob", + max_rows: 1_000_000, + optional_receipt: false, + }, + TableSpec { + name: "message_device_delivery", + columns: "blob", + max_rows: 10_000_000, + optional_receipt: false, + }, +]; + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct MigrationCheckpoint { + source_fingerprint: [u8; 32], + next_table: u8, + last_rowid: i64, + copied_counts: [u64; CHECKPOINT_TABLES], +} + +struct LegacyRow { + rowid: i64, + values: Vec, +} + +struct LegacyKeys { + fingerprint: StorageKey, + identity: StorageKey, + sessions: StorageKey, + capabilities: StorageKey, + messages: StorageKey, + queue: StorageKey, + contacts: StorageKey, + prekeys: StorageKey, + pending: StorageKey, + groups: StorageKey, + media: StorageKey, + local_metadata: StorageKey, + notes: StorageKey, + scheduled: StorageKey, + ephemeral: StorageKey, + devices: StorageKey, +} + +struct LegacyReader { + conn: Connection, + keys: LegacyKeys, + profile: KdfProfile, + has_receipt_replay: bool, + meta_fingerprint: Vec, +} + +struct SourceSnapshot { + fingerprint: [u8; 32], + counts: [u64; CHECKPOINT_TABLES], +} + +pub(crate) fn recover_missing_source(path: &Path) -> Result<()> { + if path.exists() { + return Ok(()); + } + let rollback = rollback_path(path)?; + if !rollback.is_file() { + return Ok(()); + } + atomic_replace(&rollback, path)?; + store_v2::sync_directory(parent_directory(path))?; + Ok(()) +} + +pub(crate) fn cleanup_completed_replacement(path: &Path) -> Result<()> { + let temporary = migration_path(path)?; + if temporary.exists() { + return Err(StoreError::ReplacementRecovery); + } + let rollback = rollback_path(path)?; + if rollback.exists() { + fs::remove_file(&rollback)?; + store_v2::sync_directory(parent_directory(path))?; + } + cleanup_obsolete_siblings(&temporary)?; + Ok(()) +} + +pub(crate) fn migrate_legacy( + path: &Path, + passphrase: &[u8], + lock: File, + source_database_lock: Option, +) -> Result { + let source = LegacyReader::open(path, passphrase)?; + let snapshot = source.validate_and_fingerprint()?; + ensure_workspace(path, &snapshot)?; + + let temporary = migration_path(path)?; + let mut target = if temporary.exists() { + Store::open_incomplete(&temporary, passphrase)? + } else { + let mut rng = OsRng; + let mut store = Store::create(&temporary, passphrase, source.profile, &mut rng)?; + store.metadata = store_v2::DatabaseMetadata::legacy_destination( + store.metadata.database_id, + snapshot.fingerprint, + false, + ); + let checkpoint = MigrationCheckpoint { + source_fingerprint: snapshot.fingerprint, + next_table: 0, + last_rowid: 0, + copied_counts: [0; CHECKPOINT_TABLES], + }; + let tx = store.conn.unchecked_transaction()?; + store_v2::write_metadata_with_key(&tx, &store.metadata_key, &store.metadata, &mut rng)?; + write_checkpoint_on(&store, &tx, &checkpoint, &mut rng)?; + tx.commit()?; + store + }; + + if target.metadata.source_fingerprint != Some(snapshot.fingerprint) { + return Err(StoreError::MigrationSourceChanged); + } + let incomplete = target + .metadata + .migrations + .last() + .is_some_and(|entry| !entry.completed); + if incomplete { + let checkpoint = read_checkpoint(&target)?.ok_or(StoreError::MigrationValidation)?; + if checkpoint.source_fingerprint != snapshot.fingerprint { + return Err(StoreError::MigrationSourceChanged); + } + copy_rows(&source, &target, checkpoint)?; + finish_target(&mut target, &snapshot)?; + } else { + if read_checkpoint(&target)?.is_some() { + return Err(StoreError::MigrationValidation); + } + validate_target_counts(&target, &snapshot.counts)?; + target.validate_open_state()?; + } + + sync_database_for_replacement(&target.conn)?; + target.validate_open_state()?; + drop(target); + sync_file(&temporary)?; + store_v2::sync_directory(parent_directory(path))?; + failpoint(1)?; + + let rollback = rollback_path(path)?; + if rollback.exists() { + let rollback_reader = LegacyReader::open(&rollback, passphrase)?; + let rollback_snapshot = rollback_reader.validate_and_fingerprint()?; + if rollback_snapshot.fingerprint != snapshot.fingerprint { + return Err(StoreError::ReplacementRecovery); + } + } else { + sync_database_for_replacement(&source.conn)?; + fs::copy(path, &rollback)?; + store_v2::protect_sqlite_files(&rollback)?; + sync_file(&rollback)?; + store_v2::sync_directory(parent_directory(path))?; + } + failpoint(2)?; + + drop(source); + drop(source_database_lock); + atomic_replace(&temporary, path)?; + failpoint(3)?; + store_v2::sync_directory(parent_directory(path))?; + failpoint(4)?; + + let conn = Connection::open(path)?; + let database_lock = super::acquire_database_identity_lock(path)?; + let store = match Store::open_v2_with_parts(path, passphrase, conn, database_lock, lock, false) + { + Ok(store) => store, + Err(_) => { + restore_rollback(path, &rollback)?; + return Err(StoreError::ReplacementRecovery); + } + }; + failpoint(5)?; + fs::remove_file(&rollback)?; + store_v2::sync_directory(parent_directory(path))?; + cleanup_obsolete_siblings(&temporary)?; + Ok(store) +} + +impl LegacyReader { + fn open(path: &Path, passphrase: &[u8]) -> Result { + let conn = Connection::open(path)?; + sync_database_for_replacement(&conn)?; + let has_receipt_replay = validate_released_schema(&conn)?; + let meta_count: i64 = conn.query_row("SELECT COUNT(*) FROM meta", [], |row| row.get(0))?; + if meta_count != 3 { + return Err(StoreError::UnsupportedLegacySchema); + } + let get = |key: &str| -> Result> { + conn.query_row("SELECT v FROM meta WHERE k = ?1", params![key], |row| { + row.get(0) + }) + .optional()? + .ok_or(StoreError::NotAStore) + }; + let salt_bytes = get("salt")?; + let salt: [u8; 16] = salt_bytes + .as_slice() + .try_into() + .map_err(|_| StoreError::NotAStore)?; + let kdf_bytes = get("kdf")?; + let (m_cost_kib, t_cost, p_cost): (u32, u32, u32) = + crate::decode_exact(&kdf_bytes).map_err(|_| StoreError::NotAStore)?; + let profile = KdfProfile { + m_cost_kib, + t_cost, + p_cost, + }; + let wrapped = get("wrapped_sk")?; + if wrapped.len() < 40 || wrapped.len() > 256 { + return Err(StoreError::NotAStore); + } + let kek = derive_kek(passphrase, &salt, profile)?; + let kek_key = StorageKey::from_bytes(*kek); + let master_bytes = Zeroizing::new(kek_key.open(WRAP_AD, &wrapped)?); + let master: [u8; 32] = master_bytes + .as_slice() + .try_into() + .map_err(|_| StoreError::NotAStore)?; + let master = StorageKey::from_bytes(master); + let mut meta_fingerprint = Vec::new(); + meta_fingerprint.extend_from_slice(&salt); + meta_fingerprint.extend_from_slice(&kdf_bytes); + meta_fingerprint.extend_from_slice(&wrapped); + Ok(Self { + conn, + keys: LegacyKeys { + fingerprint: master.derive(SOURCE_FINGERPRINT_DOMAIN), + identity: master.derive(b"KK-store-identity"), + sessions: master.derive(b"KK-store-sessions"), + capabilities: master.derive(b"KK-store-capabilities"), + messages: master.derive(b"KK-store-messages"), + queue: master.derive(b"KK-store-queue"), + contacts: master.derive(b"KK-store-contacts"), + prekeys: master.derive(b"KK-store-prekeys"), + pending: master.derive(b"KK-store-pending"), + groups: master.derive(b"KK-store-groups"), + media: master.derive(b"KK-store-media"), + local_metadata: master.derive(b"KK-store-local-metadata"), + notes: master.derive(b"KK-store-notes"), + scheduled: master.derive(b"KK-store-scheduled"), + ephemeral: master.derive(b"KK-store-ephemeral"), + devices: master.derive(b"KK-store-devices"), + }, + profile, + has_receipt_replay, + meta_fingerprint, + }) + } + + fn validate_and_fingerprint(&self) -> Result { + let mut state = self.keys.fingerprint.hmac_sha256(&self.meta_fingerprint); + let mut counts = [0u64; CHECKPOINT_TABLES]; + let mut total = 0u64; + for (index, spec) in TABLES.iter().enumerate() { + if spec.optional_receipt && !self.has_receipt_replay { + continue; + } + let count_sql = format!("SELECT COUNT(*) FROM {}", spec.name); + let count: i64 = self.conn.query_row(&count_sql, [], |row| row.get(0))?; + let count = u64::try_from(count).map_err(|_| StoreError::MigrationValidation)?; + if count > spec.max_rows { + return Err(StoreError::RecordBounds); + } + total = total.checked_add(count).ok_or(StoreError::RecordBounds)?; + if total > MAX_TOTAL_ROWS { + return Err(StoreError::RecordBounds); + } + counts[index] = count; + let mut after = 0i64; + loop { + let batch = read_batch(&self.conn, spec, after)?; + if batch.is_empty() { + break; + } + for row in &batch { + validate_raw_row(row)?; + let mut input = Vec::new(); + input.extend_from_slice(&state); + input.push(index as u8); + input.extend_from_slice(&row.rowid.to_be_bytes()); + for value in &row.values { + append_fingerprint_value(&mut input, value)?; + } + state = self.keys.fingerprint.hmac_sha256(&input); + after = row.rowid; + } + } + } + Ok(SourceSnapshot { + fingerprint: state, + counts, + }) + } +} + +fn copy_rows( + source: &LegacyReader, + target: &Store, + mut checkpoint: MigrationCheckpoint, +) -> Result<()> { + let mut rng = OsRng; + while usize::from(checkpoint.next_table) < TABLES.len() { + let index = usize::from(checkpoint.next_table); + let spec = &TABLES[index]; + if spec.optional_receipt && !source.has_receipt_replay { + checkpoint.next_table += 1; + checkpoint.last_rowid = 0; + let tx = target.conn.unchecked_transaction()?; + write_checkpoint_on(target, &tx, &checkpoint, &mut rng)?; + tx.commit()?; + continue; + } + let batch = read_batch(&source.conn, spec, checkpoint.last_rowid)?; + if batch.is_empty() { + checkpoint.next_table += 1; + checkpoint.last_rowid = 0; + let tx = target.conn.unchecked_transaction()?; + write_checkpoint_on(target, &tx, &checkpoint, &mut rng)?; + tx.commit()?; + continue; + } + let tx = target.conn.unchecked_transaction()?; + for row in &batch { + validate_raw_row(row)?; + migrate_row(source, target, index, row, &mut rng)?; + checkpoint.last_rowid = row.rowid; + checkpoint.copied_counts[index] = checkpoint.copied_counts[index] + .checked_add(1) + .ok_or(StoreError::RecordBounds)?; + } + write_checkpoint_on(target, &tx, &checkpoint, &mut rng)?; + tx.commit()?; + } + Ok(()) +} + +fn finish_target(target: &mut Store, snapshot: &SourceSnapshot) -> Result<()> { + let checkpoint = read_checkpoint(target)?.ok_or(StoreError::MigrationValidation)?; + if checkpoint.next_table as usize != TABLES.len() + || checkpoint.last_rowid != 0 + || checkpoint.copied_counts != snapshot.counts + { + return Err(StoreError::MigrationValidation); + } + validate_target_counts(target, &snapshot.counts)?; + let mut completed = target.metadata.clone(); + completed + .migrations + .last_mut() + .ok_or(StoreError::InvalidMigrationLedger)? + .completed = true; + let mut rng = OsRng; + let tx = target.conn.unchecked_transaction()?; + target.delete_equality_on::(&tx, &store_v2::SingletonKey)?; + store_v2::write_metadata_with_key(&tx, &target.metadata_key, &completed, &mut rng)?; + tx.commit()?; + target.metadata = completed; + let integrity: String = target + .conn + .query_row("PRAGMA integrity_check", [], |row| row.get(0))?; + if integrity != "ok" { + return Err(StoreError::MigrationValidation); + } + target.validate_open_state() +} + +fn read_checkpoint(target: &Store) -> Result> { + let Some(row) = + target.get_equality::(&store_v2::SingletonKey)? + else { + return Ok(None); + }; + row.verify_key(&store_v2::SingletonKey)?; + Ok(Some(crate::decode_exact(&row.payload)?)) +} + +pub(crate) fn validate_checkpoint_state(target: &Store) -> Result<()> { + let incomplete = target + .metadata + .migrations + .last() + .is_some_and(|entry| !entry.completed); + let mut count = 0u8; + target.validate_rows::(|row| { + count = count + .checked_add(1) + .ok_or(StoreError::MigrationValidation)?; + row.verify_key(&store_v2::SingletonKey)?; + row.verify_indexes(&store_v2::IndexKeys::none())?; + let checkpoint: MigrationCheckpoint = crate::decode_exact(&row.payload)?; + if Some(checkpoint.source_fingerprint) != target.metadata.source_fingerprint + || usize::from(checkpoint.next_table) > TABLES.len() + || checkpoint.last_rowid < 0 + || checkpoint + .copied_counts + .iter() + .any(|count| *count > MAX_TOTAL_ROWS) + { + return Err(StoreError::MigrationValidation); + } + Ok(()) + })?; + if (incomplete && count != 1) || (!incomplete && count != 0) { + return Err(StoreError::MigrationValidation); + } + Ok(()) +} + +fn write_checkpoint_on( + target: &Store, + conn: &Connection, + checkpoint: &MigrationCheckpoint, + rng: &mut OsRng, +) -> Result<()> { + let encoded = + Zeroizing::new(postcard::to_allocvec(checkpoint).map_err(|_| StoreError::Serialization)?); + target.put_equality_on::( + conn, + &store_v2::SingletonKey, + &encoded, + store_v2::IndexKeys::none(), + rng, + ) +} + +fn migrate_row( + source: &LegacyReader, + target: &Store, + table: usize, + row: &LegacyRow, + rng: &mut OsRng, +) -> Result<()> { + match table { + 0 => { + let plain = open_legacy(&source.keys.identity, b"identity", blob(row, 0)?)?; + let bytes: [u8; 64] = plain + .as_slice() + .try_into() + .map_err(|_| StoreError::Serialization)?; + target.put_identity(&Identity::from_bytes(&bytes), rng) + } + 1 => { + let peer = fixed_blob::<32>(row, 0)?; + let session = Session::unseal(blob(row, 1)?, &source.keys.sessions)?; + target.put_session(&peer, &session, rng) + } + 2 => { + let peer = fixed_blob::<32>(row, 0)?; + let plain = open_legacy(&source.keys.capabilities, b"capability", blob(row, 1)?)?; + let capability = CapabilityControl::decode(&plain)?; + target.put_capabilities(&peer, &capability, rng) + } + 3 => { + let plain = open_legacy(&source.keys.messages, b"message", blob(row, 0)?)?; + let record: MessageRecord = crate::decode_exact(&plain)?; + target.put_message(&record, rng) + } + 4 => { + let plain = open_legacy(&source.keys.queue, b"queue", blob(row, 0)?)?; + let item = Store::decode_queue_item(&plain)?; + target.queue_push(&item, rng).map(|_| ()) + } + 5 => { + let id = fixed_blob::<16>(row, 0)?; + if !target.mark_seen(&id)? { + return Err(StoreError::MigrationValidation); + } + Ok(()) + } + 6 => { + let id = fixed_blob::<16>(row, 0)?; + let plain = open_legacy(&source.keys.queue, b"receipt-replay", blob(row, 1)?)?; + let (peer, received_at): ([u8; 32], u64) = crate::decode_exact(&plain)?; + target.put_receipt_replay(&id, &peer, received_at, rng) + } + 7 => { + let peer = fixed_blob::<32>(row, 0)?; + let plain = open_legacy(&source.keys.contacts, b"contact", blob(row, 1)?)?; + let record: ContactRecord = crate::decode_exact(&plain)?; + if record.peer != peer { + return Err(StoreError::LogicalKeyMismatch); + } + target.put_contact(&record, rng) + } + 8 => { + let plain = open_legacy(&source.keys.prekeys, b"prekeys", blob(row, 0)?)?; + target.put_prekeys(&plain, rng) + } + 9 => { + let plain = open_legacy(&source.keys.pending, b"pending", blob(row, 0)?)?; + let (encoded, received_at): (Vec, u64) = crate::decode_exact(&plain)?; + let envelope = Envelope::decode(&encoded)?; + target.pending_push(&envelope, received_at, rng).map(|_| ()) + } + 10 => { + let peer = fixed_blob::<32>(row, 0)?; + target.put_reset_marker(&peer) + } + 11 => { + let group = fixed_blob::<32>(row, 0)?; + let plain = open_legacy(&source.keys.groups, b"group", blob(row, 1)?)?; + let record: GroupRecord = crate::decode_exact(&plain)?; + if record.id != group { + return Err(StoreError::LogicalKeyMismatch); + } + target.put_group(&record, rng) + } + 12 => { + let group = fixed_blob::<32>(row, 0)?; + if target.get_group(&group)?.is_none() { + return Err(StoreError::MigrationValidation); + } + let plain = open_legacy(&source.keys.groups, b"group-authority", blob(row, 1)?)?; + let record: GroupAuthorityRecord = crate::decode_exact(&plain)?; + if record.group != group { + return Err(StoreError::LogicalKeyMismatch); + } + target.put_group_authority(&record, rng) + } + 13 => { + let group = fixed_blob::<32>(row, 0)?; + let peer = fixed_blob::<32>(row, 1)?; + if target.get_group(&group)?.is_none() { + return Err(StoreError::MigrationValidation); + } + let plain = open_legacy(&source.keys.groups, b"group-chain", blob(row, 2)?)?; + target.put_group_chain(&group, &peer, &plain, rng) + } + 14 => { + let plain = open_legacy(&source.keys.groups, b"group-msg", blob(row, 0)?)?; + let record: GroupMessageRecord = crate::decode_exact(&plain)?; + target.put_group_message(&record, rng) + } + 15 => { + let local_id = fixed_blob::<16>(row, 0)?; + let plain = open_legacy(&source.keys.media, b"media-transfer", blob(row, 1)?)?; + let record: MediaTransferRecord = decode_legacy_media(&plain)?; + if record.local_id != local_id { + return Err(StoreError::LogicalKeyMismatch); + } + target.put_media_transfer(&record, rng) + } + 16 => { + let local_id = fixed_blob::<16>(row, 0)?; + let plain = open_legacy(&source.keys.media, b"media-object", blob(row, 1)?)?; + let record: MediaObjectRecord = decode_legacy_media(&plain)?; + if record.local_id != local_id + || target.get_media_transfer(&record.transfer_id)?.is_none() + { + return Err(StoreError::MigrationValidation); + } + target.put_media_object(&record, rng) + } + 17 => { + let plain = open_legacy( + &source.keys.local_metadata, + b"local-metadata", + blob(row, 0)?, + )?; + let encoded = plain + .strip_prefix(b"KLM1") + .ok_or(StoreError::Serialization)?; + let record: LocalMetadataRecord = crate::decode_exact(encoded)?; + target.put_local_metadata(&record, rng) + } + 18 => { + let plain = open_legacy(&source.keys.notes, b"note-to-self-message", blob(row, 0)?)?; + let encoded = plain + .strip_prefix(b"KNT1") + .ok_or(StoreError::Serialization)?; + let record: NoteMessageRecord = crate::decode_exact(encoded)?; + target.put_note_message(&record, rng) + } + 19 => { + let plain = open_legacy(&source.keys.scheduled, b"scheduled-message", blob(row, 0)?)?; + let record: ScheduledMessageRecord = crate::decode_exact(&plain)?; + target.put_scheduled_message(&record, rng) + } + 20 => { + let plain = open_legacy(&source.keys.ephemeral, b"ephemeral-v1", blob(row, 0)?)?; + let record: EphemeralRecord = crate::decode_exact(&plain)?; + target.put_ephemeral_record(&record, rng) + } + 21 => { + let plain = open_legacy(&source.keys.devices, b"device-state-v1", blob(row, 0)?)?; + let record: DeviceStateRecord = crate::decode_exact(&plain)?; + target.put_device_state(&record, rng) + } + 22 => { + let plain = open_legacy(&source.keys.devices, b"device-sync-v1", blob(row, 0)?)?; + if !target.put_device_sync_event(&plain, rng)? { + return Err(StoreError::MigrationValidation); + } + Ok(()) + } + 23 => { + let plain = open_legacy(&source.keys.devices, b"contact-device-v1", blob(row, 0)?)?; + let record: ContactDeviceRecord = crate::decode_exact(&plain)?; + if target.get_contact(&record.account)?.is_none() { + return Err(StoreError::MigrationValidation); + } + target.put_contact_device(&record, rng) + } + 24 => { + let plain = open_legacy( + &source.keys.devices, + b"message-device-delivery-v1", + blob(row, 0)?, + )?; + let record: MessageDeviceDeliveryRecord = crate::decode_exact(&plain)?; + if target + .row_by_unique::(&store_v2::ContentKey::new( + record.message, + ))? + .is_none() + { + return Err(StoreError::MigrationValidation); + } + target.put_message_device_delivery(&record, rng) + } + _ => Err(StoreError::MigrationValidation), + } +} + +fn decode_legacy_media(plain: &[u8]) -> Result +where + T: for<'de> Deserialize<'de>, +{ + let Some((&version, encoded)) = plain.split_first() else { + return Err(StoreError::Serialization); + }; + if version != 1 { + return Err(StoreError::UnsupportedRecordVersion); + } + crate::decode_exact(encoded) +} + +fn open_legacy( + key: &StorageKey, + associated_data: &[u8], + sealed: &[u8], +) -> Result>> { + if sealed.len() < 40 || sealed.len() > MAX_LEGACY_CIPHERTEXT { + return Err(StoreError::RecordBounds); + } + let plain = Zeroizing::new(key.open(associated_data, sealed)?); + if plain.len() > store_v2::MAX_RECORD_BYTES { + return Err(StoreError::RecordBounds); + } + Ok(plain) +} + +fn read_batch(conn: &Connection, spec: &TableSpec, after: i64) -> Result> { + let sql = format!( + "SELECT rowid, {} FROM {} WHERE rowid > ?1 ORDER BY rowid LIMIT ?2", + spec.columns, spec.name + ); + let mut statement = conn.prepare(&sql)?; + let column_count = statement.column_count(); + let rows = statement.query_map(params![after, BATCH_ROWS as i64], |row| { + let rowid = row.get::<_, i64>(0)?; + let mut values = Vec::with_capacity(column_count.saturating_sub(1)); + for column in 1..column_count { + values.push(row.get::<_, Value>(column)?); + } + Ok(LegacyRow { rowid, values }) + })?; + let mut batch = Vec::new(); + for row in rows { + batch.push(row?); + } + Ok(batch) +} + +fn validate_raw_row(row: &LegacyRow) -> Result<()> { + if row.rowid <= 0 { + return Err(StoreError::MigrationValidation); + } + for value in &row.values { + match value { + Value::Blob(bytes) if bytes.len() <= MAX_LEGACY_CIPHERTEXT => {} + Value::Integer(_) => {} + _ => return Err(StoreError::MigrationValidation), + } + } + Ok(()) +} + +fn blob(row: &LegacyRow, index: usize) -> Result<&[u8]> { + match row.values.get(index) { + Some(Value::Blob(bytes)) if bytes.len() <= MAX_LEGACY_CIPHERTEXT => Ok(bytes), + _ => Err(StoreError::MigrationValidation), + } +} + +fn fixed_blob(row: &LegacyRow, index: usize) -> Result<[u8; N]> { + blob(row, index)? + .try_into() + .map_err(|_| StoreError::MigrationValidation) +} + +fn append_fingerprint_value(output: &mut Vec, value: &Value) -> Result<()> { + match value { + Value::Integer(value) => { + output.push(1); + output.extend_from_slice(&value.to_be_bytes()); + } + Value::Blob(bytes) if bytes.len() <= MAX_LEGACY_CIPHERTEXT => { + output.push(2); + output.extend_from_slice( + &u64::try_from(bytes.len()) + .map_err(|_| StoreError::RecordBounds)? + .to_be_bytes(), + ); + output.extend_from_slice(bytes); + } + _ => return Err(StoreError::MigrationValidation), + } + Ok(()) +} + +fn validate_target_counts(target: &Store, expected: &[u64; CHECKPOINT_TABLES]) -> Result<()> { + let actual = [ + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + target.count_rows::()?, + ]; + if &actual != expected { + return Err(StoreError::MigrationValidation); + } + Ok(()) +} + +fn validate_released_schema(conn: &Connection) -> Result { + let actual = schema_objects(conn)?; + for (fixture, has_receipt) in [ + (SCHEMA_V0_1, false), + (SCHEMA_V0_2, false), + (SCHEMA_V0_3, true), + ] { + let expected = Connection::open_in_memory()?; + expected.execute_batch(fixture)?; + if actual == schema_objects(&expected)? { + return Ok(has_receipt); + } + } + Err(StoreError::UnsupportedLegacySchema) +} + +fn schema_objects(conn: &Connection) -> Result> { + let mut statement = conn.prepare( + "SELECT type, name, sql FROM sqlite_schema + WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name", + )?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?.unwrap_or_default(), + )) + })?; + let mut objects = Vec::new(); + for row in rows { + let (kind, name, sql) = row?; + objects.push((kind, name, normalize_sql(&sql))); + } + Ok(objects) +} + +fn normalize_sql(sql: &str) -> String { + sql.split_whitespace().collect::>().join(" ") +} + +fn ensure_workspace(path: &Path, snapshot: &SourceSnapshot) -> Result<()> { + let main = fs::metadata(path)?.len(); + let wal = sidecar_path(path, "-wal"); + let wal = fs::metadata(wal).map_or(0, |metadata| metadata.len()); + let source = main + .checked_add(wal) + .ok_or(StoreError::InsufficientMigrationSpace)?; + let rows = snapshot.counts.into_iter().try_fold(0u64, |total, count| { + total + .checked_add(count) + .ok_or(StoreError::InsufficientMigrationSpace) + })?; + let opaque_overhead = rows + .checked_mul(ESTIMATED_OPAQUE_ROW_OVERHEAD) + .ok_or(StoreError::InsufficientMigrationSpace)?; + let required = source + .checked_mul(3) + .and_then(|bytes| bytes.checked_add(opaque_overhead)) + .and_then(|bytes| bytes.checked_add(WORKSPACE_RESERVE)) + .ok_or(StoreError::InsufficientMigrationSpace)?; + if fs2::available_space(parent_directory(path))? < required { + return Err(StoreError::InsufficientMigrationSpace); + } + Ok(()) +} + +pub(crate) fn sync_database_for_replacement(conn: &Connection) -> Result<()> { + conn.execute_batch( + "PRAGMA synchronous = FULL; + PRAGMA wal_checkpoint(TRUNCATE); + PRAGMA journal_mode = DELETE;", + )?; + Ok(()) +} + +pub(crate) fn sync_file(path: &Path) -> Result<()> { + OpenOptions::new() + .read(true) + .write(true) + .open(path)? + .sync_all()?; + Ok(()) +} + +fn restore_rollback(path: &Path, rollback: &Path) -> Result<()> { + if !rollback.is_file() { + return Err(StoreError::ReplacementRecovery); + } + let recovery = recovery_path(path)?; + if recovery.exists() { + return Err(StoreError::ReplacementRecovery); + } + fs::copy(rollback, &recovery)?; + store_v2::protect_sqlite_files(&recovery)?; + sync_file(&recovery)?; + atomic_replace(&recovery, path)?; + store_v2::sync_directory(parent_directory(path))?; + Ok(()) +} + +pub(crate) fn cleanup_obsolete_siblings(temporary: &Path) -> Result<()> { + for suffix in ["-wal", "-shm"] { + let sidecar = sidecar_path(temporary, suffix); + if sidecar.exists() { + fs::remove_file(sidecar)?; + } + } + if let Ok(lock) = store_lock_path(temporary) { + if lock.exists() { + fs::remove_file(lock)?; + } + } + let media = media_path(temporary)?; + if media.exists() { + fs::remove_dir(&media).map_err(|_| StoreError::ReplacementRecovery)?; + } + Ok(()) +} + +fn migration_path(path: &Path) -> Result { + sibling_path(path, ".opaque-v2-migration") +} + +fn rollback_path(path: &Path) -> Result { + sibling_path(path, ".opaque-v1-rollback") +} + +fn recovery_path(path: &Path) -> Result { + sibling_path(path, ".opaque-v1-recovery") +} + +fn media_path(path: &Path) -> Result { + let file_name = path.file_name().ok_or(StoreError::NotAStore)?; + let mut name = file_name.to_os_string(); + name.push(".media"); + Ok(path.with_file_name(name)) +} + +fn sibling_path(path: &Path, suffix: &str) -> Result { + let file_name = path.file_name().ok_or(StoreError::NotAStore)?; + let mut name = file_name.to_os_string(); + name.push(suffix); + Ok(path.with_file_name(name)) +} + +fn sidecar_path(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(suffix); + PathBuf::from(name) +} + +pub(crate) fn parent_directory(path: &Path) -> &Path { + path.parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")) +} + +#[cfg(any(unix, windows))] +pub(crate) fn atomic_replace(source: &Path, destination: &Path) -> Result<()> { + atomicwrites::replace_atomic(source, destination)?; + Ok(()) +} + +#[cfg(not(any(unix, windows)))] +pub(crate) fn atomic_replace(_source: &Path, _destination: &Path) -> Result<()> { + Err(StoreError::ReplacementRecovery) +} + +#[cfg(test)] +static FAILPOINT: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +#[cfg(test)] +pub(crate) fn set_failpoint(phase: u8) { + FAILPOINT.store(phase, std::sync::atomic::Ordering::SeqCst); +} + +fn failpoint(phase: u8) -> Result<()> { + #[cfg(test)] + if FAILPOINT.load(std::sync::atomic::Ordering::SeqCst) == phase { + FAILPOINT.store(0, std::sync::atomic::Ordering::SeqCst); + return Err(StoreError::MigrationValidation); + } + let _ = phase; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::{rngs::StdRng, SeedableRng}; + + use crate::{DeliveryState, Direction}; + + const TEST_KDF: KdfProfile = KdfProfile { + m_cost_kib: 8, + t_cost: 1, + p_cost: 1, + }; + static MIGRATION_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + #[test] + fn released_schema_fixtures_are_recognized_exactly() { + for (schema, receipt) in [ + (SCHEMA_V0_1, false), + (SCHEMA_V0_2, false), + (SCHEMA_V0_3, true), + ] { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(schema).unwrap(); + assert_eq!(validate_released_schema(&conn).unwrap(), receipt); + } + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(SCHEMA_V0_3).unwrap(); + conn.execute("ALTER TABLE contacts ADD COLUMN leak BLOB", []) + .unwrap(); + assert!(matches!( + validate_released_schema(&conn), + Err(StoreError::UnsupportedLegacySchema) + )); + } + + #[test] + fn every_replacement_phase_restarts_to_one_valid_store() { + let _guard = MIGRATION_TEST_LOCK.lock().unwrap(); + for phase in 1..=5 { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join(format!("phase-{phase}.db")); + let expected = create_legacy_store(&path, SCHEMA_V0_3, phase as u64); + set_failpoint(phase); + assert!(matches!( + Store::open(&path, b"pass"), + Err(StoreError::MigrationValidation) + )); + let store = Store::open(&path, b"pass").unwrap(); + assert_eq!(store.get_identity().unwrap().unwrap().public(), expected.0); + assert_eq!(store.all_messages().unwrap(), vec![expected.1]); + assert!(!migration_path(&path).unwrap().exists()); + assert!(!rollback_path(&path).unwrap().exists()); + } + } + + #[test] + fn every_prior_public_schema_migrates_with_fresh_database_identity() { + let _guard = MIGRATION_TEST_LOCK.lock().unwrap(); + for (index, schema) in [SCHEMA_V0_1, SCHEMA_V0_2, SCHEMA_V0_3] + .into_iter() + .enumerate() + { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join(format!("released-{index}.db")); + let expected = create_legacy_store(&path, schema, 100 + index as u64); + let store = Store::open(&path, b"pass").unwrap(); + assert_ne!(store.metadata.database_id, [0; 32]); + assert_eq!(store.get_identity().unwrap().unwrap().public(), expected.0); + assert_eq!(store.all_messages().unwrap(), vec![expected.1]); + let objects = schema_objects(&store.conn).unwrap(); + assert!(objects.iter().all(|(_, name, _)| { + matches!( + name.as_str(), + "store_bootstrap" + | "store_metadata" + | "store_records" + | "store_record_locator" + | "store_record_unique" + | "store_record_index_a" + | "store_record_index_b" + | "store_record_index_c" + | "store_record_index_d" + ) + })); + } + } + + #[test] + fn malformed_legacy_rows_fail_before_active_path_replacement() { + let _guard = MIGRATION_TEST_LOCK.lock().unwrap(); + let directory = tempfile::tempdir().unwrap(); + + let typed_path = directory.path().join("wrong-type.db"); + create_legacy_store(&typed_path, SCHEMA_V0_3, 401); + let typed = Connection::open(&typed_path).unwrap(); + typed + .execute( + "INSERT INTO contacts (peer, blob) VALUES ('not-a-key', zeroblob(40))", + [], + ) + .unwrap(); + drop(typed); + assert!(matches!( + Store::open(&typed_path, b"pass"), + Err(StoreError::MigrationValidation) + )); + assert!(!store_v2::is_v2(&Connection::open(&typed_path).unwrap()).unwrap()); + assert!(!rollback_path(&typed_path).unwrap().exists()); + + let oversized_path = directory.path().join("oversized.db"); + create_legacy_store(&oversized_path, SCHEMA_V0_3, 402); + let oversized = Connection::open(&oversized_path).unwrap(); + oversized + .execute( + "INSERT INTO contacts (peer, blob) VALUES (?1, zeroblob(?2))", + params![[9u8; 32].as_slice(), (MAX_LEGACY_CIPHERTEXT + 1) as i64], + ) + .unwrap(); + drop(oversized); + assert!(matches!( + Store::open(&oversized_path, b"pass"), + Err(StoreError::MigrationValidation) + )); + assert!(!store_v2::is_v2(&Connection::open(&oversized_path).unwrap()).unwrap()); + assert!(!rollback_path(&oversized_path).unwrap().exists()); + } + + #[test] + fn logical_and_referential_mismatches_do_not_replace_the_legacy_store() { + let _guard = MIGRATION_TEST_LOCK.lock().unwrap(); + let directory = tempfile::tempdir().unwrap(); + + let logical_path = directory.path().join("logical.db"); + let seed = 403u64; + create_legacy_store(&logical_path, SCHEMA_V0_3, seed); + let logical = Connection::open(&logical_path).unwrap(); + let mut rng = StdRng::seed_from_u64(seed + 1); + let contact = ContactRecord { + peer: [4; 32], + identity: vec![1], + name: "mismatch".into(), + bundle: vec![2], + hints: Vec::new(), + verified: false, + }; + let encoded = postcard::to_allocvec(&contact).unwrap(); + let sealed = StorageKey::from_bytes([seed.wrapping_add(1) as u8; 32]) + .derive(b"KK-store-contacts") + .seal(b"contact", &encoded, &mut rng); + logical + .execute( + "INSERT INTO contacts (peer, blob) VALUES (?1, ?2)", + params![[5u8; 32].as_slice(), sealed], + ) + .unwrap(); + drop(logical); + assert!(matches!( + Store::open(&logical_path, b"pass"), + Err(StoreError::LogicalKeyMismatch) + )); + assert!(!store_v2::is_v2(&Connection::open(&logical_path).unwrap()).unwrap()); + assert!(!rollback_path(&logical_path).unwrap().exists()); + + let reference_path = directory.path().join("reference.db"); + let seed = 404u64; + create_legacy_store(&reference_path, SCHEMA_V0_3, seed); + let reference = Connection::open(&reference_path).unwrap(); + let authority = GroupAuthorityRecord { + group: [6; 32], + state_id: [7; 16], + state_payload: vec![8], + consumed_requests: Vec::new(), + }; + let encoded = postcard::to_allocvec(&authority).unwrap(); + let sealed = StorageKey::from_bytes([seed.wrapping_add(1) as u8; 32]) + .derive(b"KK-store-groups") + .seal(b"group-authority", &encoded, &mut rng); + reference + .execute( + "INSERT INTO group_authority (gid, blob) VALUES (?1, ?2)", + params![authority.group.as_slice(), sealed], + ) + .unwrap(); + drop(reference); + assert!(matches!( + Store::open(&reference_path, b"pass"), + Err(StoreError::MigrationValidation) + )); + assert!(!store_v2::is_v2(&Connection::open(&reference_path).unwrap()).unwrap()); + assert!(!rollback_path(&reference_path).unwrap().exists()); + } + + fn create_legacy_store( + path: &Path, + schema: &str, + seed: u64, + ) -> (kult_crypto::IdentityPublic, MessageRecord) { + let mut rng = StdRng::seed_from_u64(seed); + let conn = Connection::open(path).unwrap(); + conn.execute_batch(schema).unwrap(); + let salt = [seed as u8; 16]; + let master_bytes = [seed.wrapping_add(1) as u8; 32]; + let master = StorageKey::from_bytes(master_bytes); + let kek = derive_kek(b"pass", &salt, TEST_KDF).unwrap(); + let wrapped = StorageKey::from_bytes(*kek).seal(WRAP_AD, &master_bytes, &mut rng); + conn.execute( + "INSERT INTO meta (k, v) VALUES ('salt', ?1)", + params![salt.as_slice()], + ) + .unwrap(); + conn.execute( + "INSERT INTO meta (k, v) VALUES ('kdf', ?1)", + params![postcard::to_allocvec(&( + TEST_KDF.m_cost_kib, + TEST_KDF.t_cost, + TEST_KDF.p_cost, + )) + .unwrap()], + ) + .unwrap(); + conn.execute( + "INSERT INTO meta (k, v) VALUES ('wrapped_sk', ?1)", + params![wrapped], + ) + .unwrap(); + + let identity = Identity::generate(&mut rng); + let sealed = master.derive(b"KK-store-identity").seal( + b"identity", + identity.to_bytes().as_ref(), + &mut rng, + ); + conn.execute( + "INSERT INTO identity (id, blob) VALUES (1, ?1)", + params![sealed], + ) + .unwrap(); + let message = MessageRecord { + id: [seed.wrapping_add(2) as u8; 16], + peer: [seed.wrapping_add(3) as u8; 32], + direction: Direction::Inbound, + state: DeliveryState::Received, + timestamp: seed, + body: b"released fixture".to_vec(), + wire_id: None, + }; + let encoded = postcard::to_allocvec(&message).unwrap(); + let sealed = master + .derive(b"KK-store-messages") + .seal(b"message", &encoded, &mut rng); + conn.execute("INSERT INTO messages (blob) VALUES (?1)", params![sealed]) + .unwrap(); + let public = identity.public(); + drop(conn); + (public, message) + } +} diff --git a/crates/kult-store/src/note.rs b/crates/kult-store/src/note.rs index 797634b..77fd3aa 100644 --- a/crates/kult-store/src/note.rs +++ b/crates/kult-store/src/note.rs @@ -5,13 +5,12 @@ //! queue, or transport fields. use rand_core::CryptoRngCore; -use rusqlite::params; use serde::{Deserialize, Serialize}; +use zeroize::Zeroizing; -use crate::{Result, Store, StoreError}; +use crate::{decode_exact, store_v2, Result, Store, StoreError}; const RECORD_MAGIC_V1: &[u8; 4] = b"KNT1"; -const RECORD_AD: &[u8] = b"note-to-self-message"; /// Reserved conversation identity shared by node, RPC, UniFFI, and every shell. pub const NOTE_TO_SELF_CONVERSATION_ID: &str = "note_to_self"; @@ -51,34 +50,39 @@ impl Store { let mut versioned = Vec::with_capacity(RECORD_MAGIC_V1.len() + encoded.len()); versioned.extend_from_slice(RECORD_MAGIC_V1); versioned.extend_from_slice(&encoded); - let sealed = self.k_notes.seal(RECORD_AD, &versioned, rng); - self.conn.execute( - "INSERT INTO note_messages (blob) VALUES (?1)", - params![sealed], - )?; + let versioned = Zeroizing::new(versioned); + let id = store_v2::ContentKey::new(record.id); + self.append::(&id, &versioned, store_v2::IndexKeys::note(&id), rng)?; Ok(()) } /// Read note-to-self history in stable insertion order. pub fn note_messages(&self) -> Result> { - let mut statement = self - .conn - .prepare("SELECT blob FROM note_messages ORDER BY rowid_")?; - let rows = statement.query_map([], |row| row.get::<_, Vec>(0))?; let mut records = Vec::new(); - for row in rows { - let plain = self.k_notes.open(RECORD_AD, &row?)?; - let encoded = plain + for row in self.rows::()? { + let encoded = row + .payload .strip_prefix(RECORD_MAGIC_V1) .ok_or(StoreError::Serialization)?; - let (record, remainder): (NoteMessageRecord, &[u8]) = - postcard::take_from_bytes(encoded).map_err(|_| StoreError::Serialization)?; - if !remainder.is_empty() { - return Err(StoreError::Serialization); - } + let record: NoteMessageRecord = decode_exact(encoded)?; record.validate()?; + row.verify_key(&store_v2::ContentKey::new(record.id))?; records.push(record); } Ok(records) } + + pub(crate) fn validate_note_logical_rows(&self) -> Result<()> { + self.validate_rows::(|row| { + let encoded = row + .payload + .strip_prefix(RECORD_MAGIC_V1) + .ok_or(StoreError::Serialization)?; + let record: NoteMessageRecord = decode_exact(encoded)?; + record.validate()?; + let id = store_v2::ContentKey::new(record.id); + row.verify_key(&id)?; + row.verify_indexes(&store_v2::IndexKeys::note(&id)) + }) + } } diff --git a/crates/kult-store/src/opaque_tests.rs b/crates/kult-store/src/opaque_tests.rs new file mode 100644 index 0000000..a2c74fa --- /dev/null +++ b/crates/kult-store/src/opaque_tests.rs @@ -0,0 +1,461 @@ +use std::fs; + +use kult_crypto::KdfProfile; +use kult_protocol::{Envelope, EnvelopeKind}; +use rand::{rngs::StdRng, SeedableRng}; +use rusqlite::params; + +#[cfg(unix)] +use crate::store_lock_path; +use crate::store_v2::TableSpec; +use crate::{ + store_v2, ContactRecord, DeliveryState, Direction, GroupMessageRecord, GroupRecord, + MessageRecord, QueueClass, QueueItem, Store, StoreError, +}; + +const TEST_KDF: KdfProfile = KdfProfile { + m_cost_kib: 8, + t_cost: 1, + p_cost: 1, +}; + +#[test] +fn valid_ciphertext_with_wrong_inner_key_or_index_fails_closed() { + for (case, key_peer, index_peer, payload_peer) in [ + ("inner-key", [2; 32], [2; 32], [1; 32]), + ("secondary-index", [1; 32], [2; 32], [1; 32]), + ] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join(format!("{case}.db")); + let mut rng = StdRng::seed_from_u64(0x2700); + let store = Store::create(&path, b"pass", TEST_KDF, &mut rng).unwrap(); + let record = pairwise_record(7, payload_peer); + let encoded = postcard::to_allocvec(&record).unwrap(); + store + .append::( + &store_v2::MessageKey::new(key_peer, 1, record.id), + &encoded, + store_v2::IndexKeys::message( + &store_v2::ContentKey::new(record.id), + &store_v2::AccountKey::new(index_peer), + ), + &mut rng, + ) + .unwrap(); + drop(store); + assert!(matches!( + Store::open(&path, b"pass"), + Err(StoreError::LogicalKeyMismatch) + )); + } +} + +#[test] +fn copied_rows_cannot_cross_database_table_or_locator_identity() { + let directory = tempfile::tempdir().unwrap(); + let source_path = directory.path().join("source.db"); + let target_path = directory.path().join("target.db"); + let mut rng = StdRng::seed_from_u64(0x2701); + let source = Store::create(&source_path, b"pass", TEST_KDF, &mut rng).unwrap(); + source + .put_message(&pairwise_record(1, [1; 32]), &mut rng) + .unwrap(); + let copied = physical_message_row(&source); + let target = Store::create(&target_path, b"pass", TEST_KDF, &mut rng).unwrap(); + insert_physical_row(&target, store_v2::MessageRows::DOMAIN, &copied); + drop(source); + drop(target); + assert!(matches!( + Store::open(&target_path, b"pass"), + Err(StoreError::Crypto(_)) + )); + + let table_path = directory.path().join("table.db"); + let table = Store::create(&table_path, b"pass", TEST_KDF, &mut rng).unwrap(); + table + .put_message(&pairwise_record(2, [2; 32]), &mut rng) + .unwrap(); + let copied = physical_message_row(&table); + insert_physical_row(&table, store_v2::GroupMessageRows::DOMAIN, &copied); + drop(table); + assert!(matches!( + Store::open(&table_path, b"pass"), + Err(StoreError::Crypto(_)) + )); + + let locator_path = directory.path().join("locator.db"); + let locator = Store::create(&locator_path, b"pass", TEST_KDF, &mut rng).unwrap(); + locator + .put_message(&pairwise_record(3, [3; 32]), &mut rng) + .unwrap(); + locator + .conn + .execute( + "UPDATE store_records SET locator = randomblob(16) WHERE table_domain = ?1", + params![store_v2::MessageRows::DOMAIN], + ) + .unwrap(); + drop(locator); + assert!(matches!( + Store::open(&locator_path, b"pass"), + Err(StoreError::Crypto(_)) + )); +} + +#[test] +fn locked_database_has_only_opaque_identifier_columns() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("locked.db"); + let mut rng = StdRng::seed_from_u64(0x2702); + let store = Store::create(&path, b"pass", TEST_KDF, &mut rng).unwrap(); + let peer = [0xa1; 32]; + let group = [0xb2; 32]; + let message_id = [0xc3; 16]; + let group_message_id = [0xd4; 16]; + let delivery_token = [0xe5; 32]; + store + .put_contact( + &ContactRecord { + peer, + identity: vec![1, 2, 3], + name: "opaque contact".into(), + bundle: vec![4, 5], + hints: vec![vec![6]], + verified: true, + }, + &mut rng, + ) + .unwrap(); + store + .put_message( + &MessageRecord { + id: message_id, + peer, + direction: Direction::Outbound, + state: DeliveryState::Queued, + timestamp: 1, + body: b"sealed body".to_vec(), + wire_id: None, + }, + &mut rng, + ) + .unwrap(); + store + .put_group( + &GroupRecord { + id: group, + name: "opaque group".into(), + creator: peer, + members: Vec::new(), + secret: [7; 32], + prev_secret: None, + generation: 1, + sender_chain: vec![8], + sent_since_rotation: 0, + pending: Vec::new(), + }, + &mut rng, + ) + .unwrap(); + store + .put_group_message( + &GroupMessageRecord { + id: group_message_id, + group, + sender: peer, + direction: Direction::Inbound, + timestamp: 2, + body: b"group body".to_vec(), + deliveries: Vec::new(), + wire_body: None, + }, + &mut rng, + ) + .unwrap(); + let envelope = Envelope::new(EnvelopeKind::Message, delivery_token, vec![9]); + store + .queue_push( + &QueueItem { + peer, + msg_id: Some(message_id), + group_msg_id: None, + class: QueueClass::Normal, + created_at: 3, + attempts: 0, + next_attempt_at: 3, + envelope, + }, + &mut rng, + ) + .unwrap(); + + let schema = store + .conn + .prepare("SELECT name, sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name") + .unwrap() + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + }) + .unwrap() + .collect::>>() + .unwrap(); + assert_eq!(schema.len(), 10); + assert!(schema.iter().all(|(name, sql)| { + (name.starts_with("store_") || name.starts_with("sqlite_")) + && !sql.contains("peer") + && !sql.contains("group") + && !sql.contains("message_id") + && !sql.contains("token") + && !sql.contains("media_id") + })); + + let logical_identifiers: [&[u8]; 5] = [ + &peer, + &group, + &message_id, + &group_message_id, + &delivery_token, + ]; + let mut statement = store + .conn + .prepare( + "SELECT locator, unique_index, index_a, index_b, index_c, index_d + FROM store_records", + ) + .unwrap(); + let rows = statement + .query_map([], |row| { + Ok([ + Some(row.get::<_, Vec>(0)?), + row.get::<_, Option>>(1)?, + row.get::<_, Option>>(2)?, + row.get::<_, Option>>(3)?, + row.get::<_, Option>>(4)?, + row.get::<_, Option>>(5)?, + ]) + }) + .unwrap(); + for row in rows { + for physical in row.unwrap().into_iter().flatten() { + assert!(matches!(physical.len(), 16 | 32)); + assert!(logical_identifiers + .iter() + .all(|logical| physical.as_slice() != *logical)); + } + } + drop(statement); + store + .conn + .execute_batch("PRAGMA wal_checkpoint(TRUNCATE)") + .unwrap(); + drop(store); + + let mut bytes = fs::read(&path).unwrap(); + for suffix in ["-wal", "-shm"] { + let mut sidecar = path.as_os_str().to_owned(); + sidecar.push(suffix); + if let Ok(sidecar_bytes) = fs::read(std::path::PathBuf::from(sidecar)) { + bytes.extend_from_slice(&sidecar_bytes); + } + } + for logical in logical_identifiers { + assert!(!bytes.windows(logical.len()).any(|window| window == logical)); + } +} + +#[test] +fn cursors_are_stable_opaque_and_bound_to_database_conversation_and_row() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cursor.db"); + let other_path = directory.path().join("other.db"); + let mut rng = StdRng::seed_from_u64(0x2703); + let store = Store::create(&path, b"pass", TEST_KDF, &mut rng).unwrap(); + let peer = [1; 32]; + let other_peer = [2; 32]; + for value in 0..10 { + store + .put_message(&pairwise_record(value, peer), &mut rng) + .unwrap(); + store + .put_message(&pairwise_record(100 + value, other_peer), &mut rng) + .unwrap(); + } + let first = store.messages_page(&peer, None, 3).unwrap(); + assert_eq!( + first + .records + .iter() + .map(|record| record.id[0]) + .collect::>(), + vec![0, 1, 2] + ); + let cursor = first.next.unwrap(); + assert_eq!(cursor.as_bytes().len(), 57); + assert!(!cursor + .as_bytes() + .windows(peer.len()) + .any(|window| window == peer)); + let second = store.messages_page(&peer, Some(&cursor), 3).unwrap(); + assert_eq!( + second + .records + .iter() + .map(|record| record.id[0]) + .collect::>(), + vec![3, 4, 5] + ); + assert!(matches!( + store.messages_page(&other_peer, Some(&cursor), 3), + Err(StoreError::InvalidCursor) + )); + let mut forged = cursor.as_bytes().to_vec(); + forged[10] ^= 1; + assert!(matches!( + store.messages_page(&peer, Some(&crate::HistoryCursor::from_bytes(forged)), 3), + Err(StoreError::InvalidCursor) + )); + + let other = Store::create(&other_path, b"pass", TEST_KDF, &mut rng).unwrap(); + assert!(matches!( + other.messages_page(&peer, Some(&cursor), 3), + Err(StoreError::InvalidCursor) + )); + drop(other); + + assert!(store + .delete_message_record(&peer, Direction::Inbound, &[2; 16]) + .unwrap()); + assert!(matches!( + store.messages_page(&peer, Some(&cursor), 3), + Err(StoreError::InvalidCursor) + )); +} + +#[test] +fn exact_and_conversation_queries_use_the_keyed_sqlite_indexes() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("plans.db"); + let mut rng = StdRng::seed_from_u64(0x2704); + let store = Store::create(&path, b"pass", TEST_KDF, &mut rng).unwrap(); + let exact = query_plan( + &store, + "EXPLAIN QUERY PLAN SELECT blob FROM store_records + WHERE table_domain = 4 AND unique_index = randomblob(32)", + ); + assert!(exact.contains("store_record_unique"), "{exact}"); + let conversation = query_plan( + &store, + "EXPLAIN QUERY PLAN SELECT blob FROM store_records + WHERE table_domain = 4 AND index_a = randomblob(32) AND rowid_ > 0 + ORDER BY rowid_ LIMIT 65", + ); + assert!( + conversation.contains("store_record_index_a"), + "{conversation}" + ); +} + +#[cfg(unix)] +#[test] +fn sqlite_sidecars_locks_and_media_are_owner_only_on_unix() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("permissions.db"); + let mut rng = StdRng::seed_from_u64(0x2705); + let store = Store::create(&path, b"pass", TEST_KDF, &mut rng).unwrap(); + store + .put_message(&pairwise_record(1, [1; 32]), &mut rng) + .unwrap(); + let lock = store_lock_path(&path).unwrap(); + let media = store.media_dir.clone(); + for file in [&path, &lock] { + assert_eq!( + fs::metadata(file).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + for suffix in ["-wal", "-shm"] { + let mut sidecar = path.as_os_str().to_owned(); + sidecar.push(suffix); + let sidecar = std::path::PathBuf::from(sidecar); + if sidecar.exists() { + assert_eq!( + fs::metadata(sidecar).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + } + assert_eq!( + fs::metadata(media).unwrap().permissions().mode() & 0o777, + 0o700 + ); +} + +type PhysicalRow = ( + Vec, + Option>, + Option>, + Option>, + Option>, + Option>, + Vec, +); + +fn physical_message_row(store: &Store) -> PhysicalRow { + store + .conn + .query_row( + "SELECT locator, unique_index, index_a, index_b, index_c, index_d, blob + FROM store_records WHERE table_domain = ?1 LIMIT 1", + params![store_v2::MessageRows::DOMAIN], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + )) + }, + ) + .unwrap() +} + +fn insert_physical_row(store: &Store, domain: u8, row: &PhysicalRow) { + store + .conn + .execute( + "INSERT INTO store_records + (table_domain, locator, unique_index, index_a, index_b, index_c, index_d, blob) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![domain, &row.0, &row.1, &row.2, &row.3, &row.4, &row.5, &row.6], + ) + .unwrap(); +} + +fn pairwise_record(value: u8, peer: [u8; 32]) -> MessageRecord { + MessageRecord { + id: [value; 16], + peer, + direction: Direction::Inbound, + state: DeliveryState::Received, + timestamp: u64::from(value), + body: vec![value], + wire_id: None, + } +} + +fn query_plan(store: &Store, sql: &str) -> String { + store + .conn + .prepare(sql) + .unwrap() + .query_map([], |row| row.get::<_, String>(3)) + .unwrap() + .collect::>>() + .unwrap() + .join("\n") +} diff --git a/crates/kult-store/src/scale_bench.rs b/crates/kult-store/src/scale_bench.rs new file mode 100644 index 0000000..0136994 --- /dev/null +++ b/crates/kult-store/src/scale_bench.rs @@ -0,0 +1,273 @@ +use std::fs; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, Instant}; + +use kult_crypto::{derive_kek, Identity, StorageKey, KDF_PROFILE_MOBILE}; +use rand::{rngs::StdRng, SeedableRng}; +use rusqlite::{params, Connection}; + +use crate::{DeliveryState, Direction, MessageRecord, Store, LEGACY_SCHEMA_CURRENT, WRAP_AD}; + +struct Budgets { + migration: Duration, + unlock: Duration, + page: Duration, + exact_edit: Duration, + exact_delete: Duration, + migration_peak_delta_bytes: u64, + database_bytes: u64, +} + +#[test] +#[ignore = "large storage qualification; run through scripts/store-scale-gate.sh"] +fn opaque_store_scale_budget() { + let messages = std::env::var("KOMMS_STORE_BENCH_MESSAGES") + .ok() + .and_then(|value| value.parse::().ok()) + .expect("KOMMS_STORE_BENCH_MESSAGES must be 100000 or 1000000"); + assert!(matches!(messages, 100_000 | 1_000_000)); + let budgets = budgets(messages); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join(format!("opaque-scale-{messages}.db")); + let peer = [0x51; 32]; + create_large_legacy_store(&path, messages, peer); + + let rss_before = resident_bytes(); + let running = Arc::new(AtomicBool::new(true)); + let peak_rss = Arc::new(AtomicU64::new(rss_before.unwrap_or(0))); + let monitor = rss_before.map(|_| { + let running = Arc::clone(&running); + let peak_rss = Arc::clone(&peak_rss); + thread::spawn(move || { + while running.load(Ordering::Relaxed) { + if let Some(current) = resident_bytes() { + peak_rss.fetch_max(current, Ordering::Relaxed); + } + thread::sleep(Duration::from_millis(10)); + } + }) + }); + + let migration_started = Instant::now(); + let store = Store::open(&path, b"scale-passphrase").unwrap(); + let migration_elapsed = migration_started.elapsed(); + running.store(false, Ordering::Relaxed); + if let Some(monitor) = monitor { + monitor.join().unwrap(); + } + let migration_peak_delta = + rss_before.map(|baseline| peak_rss.load(Ordering::Relaxed).saturating_sub(baseline)); + assert!( + migration_elapsed <= budgets.migration, + "migration took {migration_elapsed:?}, budget {:?}", + budgets.migration + ); + if let Some(delta) = migration_peak_delta { + assert!( + delta <= budgets.migration_peak_delta_bytes, + "migration peak RSS delta was {delta} bytes, budget {}", + budgets.migration_peak_delta_bytes + ); + } + drop(store); + + let unlock_started = Instant::now(); + let store = Store::open(&path, b"scale-passphrase").unwrap(); + let unlock_elapsed = unlock_started.elapsed(); + assert!( + unlock_elapsed <= budgets.unlock, + "unlock took {unlock_elapsed:?}, budget {:?}", + budgets.unlock + ); + + let page_started = Instant::now(); + let first_page = store.messages_page(&peer, None, 64).unwrap(); + let page_elapsed = page_started.elapsed(); + assert_eq!(first_page.records.len(), 64); + assert!(first_page.next.is_some()); + assert!( + page_elapsed <= budgets.page, + "page lookup took {page_elapsed:?}, budget {:?}", + budgets.page + ); + + let edit_value = messages / 2; + let edited = message_record(edit_value, peer); + let edited = MessageRecord { + body: b"edited by exact opaque id lookup".to_vec(), + ..edited + }; + let mut rng = StdRng::seed_from_u64(0x0ade_0027); + let edit_started = Instant::now(); + assert!(store.update_message(&edited, &mut rng).unwrap()); + let edit_elapsed = edit_started.elapsed(); + assert!( + edit_elapsed <= budgets.exact_edit, + "exact edit took {edit_elapsed:?}, budget {:?}", + budgets.exact_edit + ); + let deleted = message_record(messages - 1, peer); + let delete_started = Instant::now(); + assert!(store + .delete_message_record(&deleted.peer, deleted.direction, &deleted.id) + .unwrap()); + let delete_elapsed = delete_started.elapsed(); + assert!( + delete_elapsed <= budgets.exact_delete, + "exact delete took {delete_elapsed:?}, budget {:?}", + budgets.exact_delete + ); + + let database_bytes = sqlite_bytes(&path); + assert!( + database_bytes <= budgets.database_bytes, + "database uses {database_bytes} bytes, budget {}", + budgets.database_bytes + ); + println!( + "STORE_SCALE_RESULT messages={messages} migration_ms={} unlock_ms={} page_us={} \ + edit_us={} delete_us={} migration_peak_delta_bytes={} database_bytes={} \ + migration_budget_ms={} unlock_budget_ms={} page_budget_us={} edit_budget_us={} \ + delete_budget_us={} memory_budget_bytes={} database_budget_bytes={}", + migration_elapsed.as_millis(), + unlock_elapsed.as_millis(), + page_elapsed.as_micros(), + edit_elapsed.as_micros(), + delete_elapsed.as_micros(), + migration_peak_delta.unwrap_or(0), + database_bytes, + budgets.migration.as_millis(), + budgets.unlock.as_millis(), + budgets.page.as_micros(), + budgets.exact_edit.as_micros(), + budgets.exact_delete.as_micros(), + budgets.migration_peak_delta_bytes, + budgets.database_bytes, + ); +} + +fn budgets(messages: u64) -> Budgets { + match messages { + 100_000 => Budgets { + migration: Duration::from_secs(180), + unlock: Duration::from_secs(30), + page: Duration::from_millis(250), + exact_edit: Duration::from_millis(250), + exact_delete: Duration::from_millis(250), + migration_peak_delta_bytes: 512 * 1024 * 1024, + database_bytes: 64 * 1024 * 1024 + messages * 1_024, + }, + 1_000_000 => Budgets { + migration: Duration::from_secs(1_800), + unlock: Duration::from_secs(180), + page: Duration::from_millis(500), + exact_edit: Duration::from_millis(500), + exact_delete: Duration::from_millis(500), + migration_peak_delta_bytes: 768 * 1024 * 1024, + database_bytes: 64 * 1024 * 1024 + messages * 1_024, + }, + _ => unreachable!(), + } +} + +fn create_large_legacy_store(path: &std::path::Path, messages: u64, peer: [u8; 32]) { + let mut rng = StdRng::seed_from_u64(0x0005_ca1e_0027); + let conn = Connection::open(path).unwrap(); + conn.execute_batch(LEGACY_SCHEMA_CURRENT).unwrap(); + conn.pragma_update(None, "synchronous", "OFF").unwrap(); + conn.pragma_update(None, "journal_mode", "MEMORY").unwrap(); + let salt = [0x27; 16]; + let master_bytes = [0xa7; 32]; + let master = StorageKey::from_bytes(master_bytes); + let kek = derive_kek(b"scale-passphrase", &salt, KDF_PROFILE_MOBILE).unwrap(); + let wrapped = StorageKey::from_bytes(*kek).seal(WRAP_AD, &master_bytes, &mut rng); + conn.execute( + "INSERT INTO meta (k, v) VALUES ('salt', ?1)", + params![salt.as_slice()], + ) + .unwrap(); + conn.execute( + "INSERT INTO meta (k, v) VALUES ('kdf', ?1)", + params![postcard::to_allocvec(&( + KDF_PROFILE_MOBILE.m_cost_kib, + KDF_PROFILE_MOBILE.t_cost, + KDF_PROFILE_MOBILE.p_cost, + )) + .unwrap()], + ) + .unwrap(); + conn.execute( + "INSERT INTO meta (k, v) VALUES ('wrapped_sk', ?1)", + params![wrapped], + ) + .unwrap(); + let identity = Identity::generate(&mut rng); + let sealed_identity = master.derive(b"KK-store-identity").seal( + b"identity", + identity.to_bytes().as_ref(), + &mut rng, + ); + conn.execute( + "INSERT INTO identity (id, blob) VALUES (1, ?1)", + params![sealed_identity], + ) + .unwrap(); + + let message_key = master.derive(b"KK-store-messages"); + for start in (0..messages).step_by(4_096) { + let end = (start + 4_096).min(messages); + let tx = conn.unchecked_transaction().unwrap(); + { + let mut insert = tx + .prepare_cached("INSERT INTO messages (blob) VALUES (?1)") + .unwrap(); + for value in start..end { + let encoded = postcard::to_allocvec(&message_record(value, peer)).unwrap(); + let sealed = message_key.seal(b"message", &encoded, &mut rng); + insert.execute(params![sealed]).unwrap(); + } + } + tx.commit().unwrap(); + } + drop(conn); +} + +fn message_record(value: u64, peer: [u8; 32]) -> MessageRecord { + let mut id = [0u8; 16]; + id[..8].copy_from_slice(&value.to_be_bytes()); + id[8..].copy_from_slice(&(!value).to_be_bytes()); + MessageRecord { + id, + peer, + direction: Direction::Inbound, + state: DeliveryState::Received, + timestamp: value, + body: vec![0x42; 32], + wire_id: None, + } +} + +fn sqlite_bytes(path: &std::path::Path) -> u64 { + let main = fs::metadata(path).unwrap().len(); + let mut wal_name = path.as_os_str().to_owned(); + wal_name.push("-wal"); + main + fs::metadata(std::path::PathBuf::from(wal_name)).map_or(0, |value| value.len()) +} + +#[cfg(target_os = "linux")] +fn resident_bytes() -> Option { + let status = fs::read_to_string("/proc/self/status").ok()?; + let line = status.lines().find(|line| line.starts_with("VmRSS:"))?; + line.split_whitespace() + .nth(1)? + .parse::() + .ok()? + .checked_mul(1_024) +} + +#[cfg(not(target_os = "linux"))] +fn resident_bytes() -> Option { + None +} diff --git a/crates/kult-store/src/scheduled.rs b/crates/kult-store/src/scheduled.rs index dead7ec..f859135 100644 --- a/crates/kult-store/src/scheduled.rs +++ b/crates/kult-store/src/scheduled.rs @@ -5,10 +5,10 @@ //! can safely edit or cancel it beforehand. use rand_core::CryptoRngCore; -use rusqlite::params; use serde::{Deserialize, Serialize}; +use zeroize::Zeroizing; -use crate::{Result, Store, StoreError}; +use crate::{decode_exact, store_v2, Result, Store, StoreError}; /// A conversation that can receive scheduled text. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -41,35 +41,40 @@ impl Store { record: &ScheduledMessageRecord, rng: &mut impl CryptoRngCore, ) -> Result<()> { - let plain = postcard::to_allocvec(record).map_err(|_| StoreError::Serialization)?; - let sealed = self.k_scheduled.seal(b"scheduled-message", &plain, rng); - self.conn.execute( - "INSERT INTO scheduled_messages (blob) VALUES (?1)", - params![sealed], + let plain = + Zeroizing::new(postcard::to_allocvec(record).map_err(|_| StoreError::Serialization)?); + self.put_equality::( + &store_v2::ContentKey::new(record.id), + &plain, + store_v2::IndexKeys::none(), + rng, )?; Ok(()) } /// All scheduled messages in creation order. pub fn scheduled_messages(&self) -> Result> { - let mut stmt = self - .conn - .prepare("SELECT blob FROM scheduled_messages ORDER BY rowid_")?; - let rows = stmt.query_map([], |row| row.get::<_, Vec>(0))?; let mut out = Vec::new(); - for row in rows { - let plain = self.k_scheduled.open(b"scheduled-message", &row?)?; - out.push(postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?); + for row in self.rows::()? { + let record: ScheduledMessageRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::ContentKey::new(record.id))?; + out.push(record); } Ok(out) } /// Load one scheduled message by stable id. pub fn get_scheduled_message(&self, id: &[u8; 16]) -> Result> { - Ok(self - .scheduled_messages()? - .into_iter() - .find(|record| &record.id == id)) + let key = store_v2::ContentKey::new(*id); + let Some(row) = self.get_equality::(&key)? else { + return Ok(None); + }; + row.verify_key(&key)?; + let record: ScheduledMessageRecord = decode_exact(&row.payload)?; + if record.id != *id { + return Err(StoreError::LogicalKeyMismatch); + } + Ok(Some(record)) } /// Replace the scheduled message with the same id. Returns whether it @@ -79,52 +84,28 @@ impl Store { record: &ScheduledMessageRecord, rng: &mut impl CryptoRngCore, ) -> Result { - let mut stmt = self - .conn - .prepare("SELECT rowid_, blob FROM scheduled_messages ORDER BY rowid_")?; - let rows = stmt.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - })?; - for row in rows { - let (rowid, sealed) = row?; - let plain = self.k_scheduled.open(b"scheduled-message", &sealed)?; - let stored: ScheduledMessageRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if stored.id == record.id { - let plain = postcard::to_allocvec(record).map_err(|_| StoreError::Serialization)?; - let sealed = self.k_scheduled.seal(b"scheduled-message", &plain, rng); - self.conn.execute( - "UPDATE scheduled_messages SET blob = ?2 WHERE rowid_ = ?1", - params![rowid, sealed], - )?; - return Ok(true); - } + let key = store_v2::ContentKey::new(record.id); + if self + .get_equality::(&key)? + .is_none() + { + return Ok(false); } - Ok(false) + self.put_scheduled_message(record, rng)?; + Ok(true) } /// Delete a scheduled message. Returns whether it still existed. pub fn delete_scheduled_message(&self, id: &[u8; 16]) -> Result { - let mut stmt = self - .conn - .prepare("SELECT rowid_, blob FROM scheduled_messages ORDER BY rowid_")?; - let rows = stmt.query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - })?; - for row in rows { - let (rowid, sealed) = row?; - let plain = self.k_scheduled.open(b"scheduled-message", &sealed)?; - let stored: ScheduledMessageRecord = - postcard::from_bytes(&plain).map_err(|_| StoreError::Serialization)?; - if &stored.id == id { - self.conn.execute( - "DELETE FROM scheduled_messages WHERE rowid_ = ?1", - params![rowid], - )?; - return Ok(true); - } - } - Ok(false) + self.delete_equality::(&store_v2::ContentKey::new(*id)) + } + + pub(crate) fn validate_scheduled_logical_rows(&self) -> Result<()> { + self.validate_rows::(|row| { + let record: ScheduledMessageRecord = decode_exact(&row.payload)?; + row.verify_key(&store_v2::ContentKey::new(record.id))?; + row.verify_indexes(&store_v2::IndexKeys::none()) + }) } } diff --git a/crates/kult-store/src/store_v2.rs b/crates/kult-store/src/store_v2.rs index fa0b7c1..6d4a206 100644 --- a/crates/kult-store/src/store_v2.rs +++ b/crates/kult-store/src/store_v2.rs @@ -1,600 +1,1029 @@ -//! Inactive ADR-0027 migration-target foundation. +//! ADR-0027 opaque, versioned, row-bound storage primitives. //! -//! The public [`crate::Store`] continues to open only the complete legacy -//! schema. This module gives the future all-table sibling migration one -//! independently testable destination format without rewriting any existing -//! user database or allowing callers to assemble raw index/row domains. +//! Every logical table is represented inside one strict SQLite record table. +//! Callers select tables and lookup indexes through marker types; raw table, +//! index, key-derivation, and associated-data domains remain private here. -use std::collections::BTreeSet; +#[cfg(unix)] use std::fs::File; use std::path::Path; use rand_core::CryptoRngCore; -use rusqlite::{ - params, Connection, ErrorCode, OpenFlags, OptionalExtension, Transaction, TransactionBehavior, -}; +use rusqlite::{params, Connection, ErrorCode, OptionalExtension}; use serde::{Deserialize, Serialize}; use zeroize::Zeroizing; -use kult_crypto::StorageKey; +use kult_crypto::{KdfProfile, StorageKey}; -use crate::{acquire_database_identity_lock, acquire_store_lock, Result, StoreError}; +use crate::{Result, Store, StoreError}; -const APPLICATION_ID: u32 = 0x4b53_5632; -const SCHEMA_VERSION: u32 = 2; +pub(crate) const APPLICATION_ID: u32 = 0x4b53_5632; +pub(crate) const SCHEMA_VERSION: u32 = 2; const METADATA_RECORD_VERSION: u8 = 1; const LOGICAL_RECORD_VERSION: u8 = 1; const MAX_MIGRATIONS: usize = 64; -const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024; +pub(crate) const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024; +const MAX_CANONICAL_KEY_BYTES: usize = 2 * 1024; const ROW_LOCATOR_ATTEMPTS: usize = 16; +pub(crate) const MAX_PAGE_SIZE: usize = 512; const METADATA_AD: &[u8] = b"Komms-Store-Metadata-v2"; const INDEX_INPUT_DOMAIN: &[u8] = b"Komms-Store-Index-v2"; const INDEX_KEY_DOMAIN: &[u8] = b"Komms-Store-Index-Key-v2"; const ROW_KEY_DOMAIN: &[u8] = b"Komms-Store-Row-Key-v2"; const ROW_AD_DOMAIN: &[u8] = b"Komms-Store-Row-v2"; -const FOUNDATION_MIGRATION_ID: [u8; 16] = *b"opaque-store-v2!"; - -const METADATA_TABLE_SCHEMA: &str = "CREATE TABLE store_v2_metadata ( +const CURSOR_KEY_DOMAIN: &[u8] = b"Komms-Store-Cursor-Key-v2"; +const CURSOR_MASK_DOMAIN: &[u8] = b"Komms-Store-Cursor-Mask-v2"; +const CURSOR_TAG_DOMAIN: &[u8] = b"Komms-Store-Cursor-Tag-v2"; +const FRESH_MIGRATION_ID: [u8; 16] = *b"opaque-store-v2!"; +pub(crate) const LEGACY_MIGRATION_ID: [u8; 16] = *b"legacy-schema-v1"; +pub(crate) const OPAQUE_MIGRATION_ID: [u8; 16] = *b"opaque-all-v2!!!"; + +pub(crate) const BOOTSTRAP_TABLE_SCHEMA: &str = "CREATE TABLE store_bootstrap ( + id INTEGER PRIMARY KEY CHECK (id = 1), + salt BLOB NOT NULL CHECK (length(salt) = 16), + kdf BLOB NOT NULL, + wrapped_sk BLOB NOT NULL +) STRICT"; +pub(crate) const METADATA_TABLE_SCHEMA: &str = "CREATE TABLE store_metadata ( id INTEGER PRIMARY KEY CHECK (id = 1), blob BLOB NOT NULL ) STRICT"; -const RECORDS_TABLE_SCHEMA: &str = "CREATE TABLE store_v2_records ( - rowid_ INTEGER PRIMARY KEY, +pub(crate) const RECORDS_TABLE_SCHEMA: &str = "CREATE TABLE store_records ( + rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, table_domain INTEGER NOT NULL, locator BLOB NOT NULL CHECK (length(locator) IN (16, 32)), + unique_index BLOB CHECK (unique_index IS NULL OR length(unique_index) = 32), + index_a BLOB CHECK (index_a IS NULL OR length(index_a) = 32), + index_b BLOB CHECK (index_b IS NULL OR length(index_b) = 32), + index_c BLOB CHECK (index_c IS NULL OR length(index_c) = 32), + index_d BLOB CHECK (index_d IS NULL OR length(index_d) = 32), blob BLOB NOT NULL ) STRICT"; -const RECORD_LOCATOR_INDEX_SCHEMA: &str = "CREATE UNIQUE INDEX store_v2_record_locator - ON store_v2_records (table_domain, locator)"; +pub(crate) const RECORD_LOCATOR_INDEX_SCHEMA: &str = + "CREATE UNIQUE INDEX store_record_locator ON store_records (table_domain, locator)"; +pub(crate) const RECORD_UNIQUE_INDEX_SCHEMA: &str = "CREATE UNIQUE INDEX store_record_unique + ON store_records (table_domain, unique_index) WHERE unique_index IS NOT NULL"; +pub(crate) const RECORD_INDEX_A_SCHEMA: &str = + "CREATE INDEX store_record_index_a ON store_records (table_domain, index_a, rowid_)"; +pub(crate) const RECORD_INDEX_B_SCHEMA: &str = + "CREATE INDEX store_record_index_b ON store_records (table_domain, index_b, rowid_)"; +pub(crate) const RECORD_INDEX_C_SCHEMA: &str = + "CREATE INDEX store_record_index_c ON store_records (table_domain, index_c, rowid_)"; +pub(crate) const RECORD_INDEX_D_SCHEMA: &str = + "CREATE INDEX store_record_index_d ON store_records (table_domain, index_d, rowid_)"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -struct MigrationEntry { - id: [u8; 16], - from_version: u32, - to_version: u32, - completed: bool, +pub(crate) struct MigrationEntry { + pub(crate) id: [u8; 16], + pub(crate) from_version: u32, + pub(crate) to_version: u32, + pub(crate) completed: bool, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -struct DatabaseMetadata { - database_id: [u8; 32], - schema_version: u32, - migrations: Vec, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum LocatorKind { - Equality, - Random, +pub(crate) struct DatabaseMetadata { + pub(crate) database_id: [u8; 32], + pub(crate) schema_version: u32, + pub(crate) migrations: Vec, + pub(crate) source_fingerprint: Option<[u8; 32]>, } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -#[repr(u8)] -/// Closed set of v2 table domains; callers never provide raw domain bytes. -enum TableDomain { - /// Contact equality records. - Contacts = 1, - /// Session-scoped capability equality records. - Capabilities = 2, - /// Append-only pairwise history records. - Messages = 3, -} - -impl TableDomain { - fn from_sql(value: i64) -> Result { - match value { - 1 => Ok(Self::Contacts), - 2 => Ok(Self::Capabilities), - 3 => Ok(Self::Messages), - _ => Err(StoreError::SchemaMismatch), - } - } - - fn locator_kind(self) -> LocatorKind { - match self { - Self::Contacts | Self::Capabilities => LocatorKind::Equality, - Self::Messages => LocatorKind::Random, +impl DatabaseMetadata { + pub(crate) fn fresh(database_id: [u8; 32]) -> Self { + Self { + database_id, + schema_version: SCHEMA_VERSION, + migrations: vec![MigrationEntry { + id: FRESH_MIGRATION_ID, + from_version: 0, + to_version: SCHEMA_VERSION, + completed: true, + }], + source_fingerprint: None, } } - fn key_kind(self) -> u8 { - match self { - Self::Contacts | Self::Capabilities => AccountKey::KIND, - Self::Messages => MessageKey::KIND, + pub(crate) fn legacy_destination( + database_id: [u8; 32], + source_fingerprint: [u8; 32], + completed: bool, + ) -> Self { + Self { + database_id, + schema_version: SCHEMA_VERSION, + migrations: vec![ + MigrationEntry { + id: LEGACY_MIGRATION_ID, + from_version: 0, + to_version: 1, + completed: true, + }, + MigrationEntry { + id: OPAQUE_MIGRATION_ID, + from_version: 1, + to_version: SCHEMA_VERSION, + completed, + }, + ], + source_fingerprint: Some(source_fingerprint), } } +} - fn expected_key_len(self) -> usize { - match self { - Self::Contacts | Self::Capabilities => AccountKey::ENCODED_LEN, - Self::Messages => MessageKey::ENCODED_LEN, - } - } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum LocatorKind { + Equality, + Random, } -/// Closed canonical logical-key encoding used inside sealed records. -trait CanonicalLogicalKey { - /// Encode the type byte followed by the key's fixed-width fields. +pub(crate) trait LogicalKey { + const KIND: u8; + fn encode(&self) -> Vec; + + fn validate_encoded(encoded: &[u8]) -> bool; } -/// Table marker binding one logical key type to one fixed table domain. -trait RecordTable { - /// Exact logical key accepted by this table. - type Key: CanonicalLogicalKey; +pub(crate) trait TableSpec { + type Key: LogicalKey; - /// Fixed domain used for key derivation, SQLite classification, and AD. - const DOMAIN: TableDomain; + const DOMAIN: u8; + const LOCATOR_KIND: LocatorKind; } -/// Marker for a table whose locator is a keyed equality index. -trait EqualityTable: RecordTable {} -/// Marker for a table whose locator is freshly random for every row. -trait AppendTable: RecordTable {} +macro_rules! fixed_key { + ($name:ident, $kind:expr, $size:expr) => { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(crate) struct $name(pub(crate) [u8; $size]); -/// Typed account identity used by v2 equality-index tables. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct AccountKey([u8; 32]); + impl $name { + pub(crate) fn new(value: [u8; $size]) -> Self { + Self(value) + } -impl AccountKey { - const KIND: u8 = 1; - const ENCODED_LEN: usize = 1 + 32; + pub(crate) fn decode(encoded: &[u8]) -> Result { + if !Self::validate_encoded(encoded) { + return Err(StoreError::LogicalKeyMismatch); + } + Ok(Self( + encoded[1..] + .try_into() + .map_err(|_| StoreError::LogicalKeyMismatch)?, + )) + } - /// Bind an exact account public key to the account-key logical domain. - pub fn new(value: [u8; 32]) -> Self { - Self(value) - } + pub(crate) fn value(&self) -> &[u8; $size] { + &self.0 + } + } + + impl LogicalKey for $name { + const KIND: u8 = $kind; + + fn encode(&self) -> Vec { + let mut encoded = Vec::with_capacity(1 + $size); + encoded.push(Self::KIND); + encoded.extend_from_slice(&self.0); + encoded + } + + fn validate_encoded(encoded: &[u8]) -> bool { + encoded.len() == 1 + $size && encoded.first() == Some(&Self::KIND) + } + } + }; } -impl CanonicalLogicalKey for AccountKey { +fixed_key!(AccountKey, 1, 32); +fixed_key!(GroupKey, 2, 32); +fixed_key!(ContentKey, 3, 16); +fixed_key!(LocalIdKey, 4, 16); +fixed_key!(DigestKey, 5, 32); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SingletonKey; + +impl LogicalKey for SingletonKey { + const KIND: u8 = 6; + fn encode(&self) -> Vec { - let mut encoded = Vec::with_capacity(Self::ENCODED_LEN); - encoded.push(Self::KIND); - encoded.extend_from_slice(&self.0); - encoded + vec![Self::KIND, 1] + } + + fn validate_encoded(encoded: &[u8]) -> bool { + encoded == [Self::KIND, 1] } } -/// Typed immutable message identity used inside append-only v2 records. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct MessageKey { - conversation: [u8; 32], - message_id: [u8; 16], +pub(crate) struct MessageKey { + peer: [u8; 32], + direction: u8, + id: [u8; 16], } impl MessageKey { - const KIND: u8 = 2; - const ENCODED_LEN: usize = 1 + 32 + 16; - - /// Bind one message id to its exact pairwise conversation. - pub fn new(conversation: [u8; 32], message_id: [u8; 16]) -> Self { + pub(crate) fn new(peer: [u8; 32], direction: u8, id: [u8; 16]) -> Self { Self { - conversation, - message_id, + peer, + direction, + id, } } } -impl CanonicalLogicalKey for MessageKey { +impl LogicalKey for MessageKey { + const KIND: u8 = 7; + fn encode(&self) -> Vec { - let mut encoded = Vec::with_capacity(Self::ENCODED_LEN); + let mut encoded = Vec::with_capacity(50); encoded.push(Self::KIND); - encoded.extend_from_slice(&self.conversation); - encoded.extend_from_slice(&self.message_id); + encoded.extend_from_slice(&self.peer); + encoded.push(self.direction); + encoded.extend_from_slice(&self.id); encoded } -} -/// Contact-table marker; its physical locator is a keyed account index. -struct ContactRecords; + fn validate_encoded(encoded: &[u8]) -> bool { + encoded.len() == 50 + && encoded.first() == Some(&Self::KIND) + && matches!(encoded.get(33), Some(0 | 1)) + } +} -impl RecordTable for ContactRecords { - type Key = AccountKey; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct GroupMessageKey { + group: [u8; 32], + sender: [u8; 32], + direction: u8, + id: [u8; 16], +} - const DOMAIN: TableDomain = TableDomain::Contacts; +impl GroupMessageKey { + pub(crate) fn new(group: [u8; 32], sender: [u8; 32], direction: u8, id: [u8; 16]) -> Self { + Self { + group, + sender, + direction, + id, + } + } } -impl EqualityTable for ContactRecords {} +impl LogicalKey for GroupMessageKey { + const KIND: u8 = 8; -/// Capability-table marker; it derives an index unrelated to contacts. -struct CapabilityRecords; + fn encode(&self) -> Vec { + let mut encoded = Vec::with_capacity(82); + encoded.push(Self::KIND); + encoded.extend_from_slice(&self.group); + encoded.extend_from_slice(&self.sender); + encoded.push(self.direction); + encoded.extend_from_slice(&self.id); + encoded + } -impl RecordTable for CapabilityRecords { - type Key = AccountKey; + fn validate_encoded(encoded: &[u8]) -> bool { + encoded.len() == 82 + && encoded.first() == Some(&Self::KIND) + && matches!(encoded.get(65), Some(0 | 1)) + } +} - const DOMAIN: TableDomain = TableDomain::Capabilities; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct GroupMemberKey { + group: [u8; 32], + peer: [u8; 32], } -impl EqualityTable for CapabilityRecords {} +impl GroupMemberKey { + pub(crate) fn new(group: [u8; 32], peer: [u8; 32]) -> Self { + Self { group, peer } + } -/// Pairwise history marker; append rows use random locators. -struct MessageRecords; + pub(crate) fn decode(encoded: &[u8]) -> Result { + if !Self::validate_encoded(encoded) { + return Err(StoreError::LogicalKeyMismatch); + } + let group = encoded[1..33] + .try_into() + .map_err(|_| StoreError::LogicalKeyMismatch)?; + let peer = encoded[33..65] + .try_into() + .map_err(|_| StoreError::LogicalKeyMismatch)?; + Ok(Self { group, peer }) + } -impl RecordTable for MessageRecords { - type Key = MessageKey; + pub(crate) fn group(&self) -> &[u8; 32] { + &self.group + } - const DOMAIN: TableDomain = TableDomain::Messages; + pub(crate) fn peer(&self) -> &[u8; 32] { + &self.peer + } } -impl AppendTable for MessageRecords {} +impl LogicalKey for GroupMemberKey { + const KIND: u8 = 9; -/// Opaque random locator returned for one append-only row. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct RowLocator([u8; 16]); + fn encode(&self) -> Vec { + let mut encoded = Vec::with_capacity(65); + encoded.push(Self::KIND); + encoded.extend_from_slice(&self.group); + encoded.extend_from_slice(&self.peer); + encoded + } -impl RowLocator { - /// The locator bytes are safe to pass back only to this database. - pub fn as_bytes(&self) -> &[u8; 16] { - &self.0 + fn validate_encoded(encoded: &[u8]) -> bool { + encoded.len() == 65 && encoded.first() == Some(&Self::KIND) } } -struct DecodedRecord<'a> { - logical_key: &'a [u8], - payload: &'a [u8], +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct AccountDeviceKey { + account: [u8; 32], + device: [u8; 32], } -/// Unconstructable outside `kult-store`; gates the inactive migration target. -/// -/// The complete sibling migrator will construct this token inside this module -/// when it is ready to activate the destination atomically. -pub struct MigrationAuthority { - private: (), -} - -/// A complete but inactive v2 destination used by the future sibling migration. -/// -/// It deliberately takes an already available master key. Passphrase wrapping, -/// backup restore, and atomic path replacement remain responsibilities of the -/// all-table migration and are not activated by this foundation. -pub struct MigrationTarget { - conn: Connection, - metadata: DatabaseMetadata, - index_root: StorageKey, - row_root: StorageKey, - _database_lock: Option, - _lock: File, -} - -impl MigrationTarget { - /// Create one empty v2 migration destination without touching a legacy store. - pub fn create( - authority: &MigrationAuthority, - path: &Path, - master: &StorageKey, - rng: &mut impl CryptoRngCore, - ) -> Result { - let () = authority.private; - let lock = acquire_store_lock(path)?; - let conn = Connection::open_with_flags( - path, - OpenFlags::SQLITE_OPEN_READ_WRITE - | OpenFlags::SQLITE_OPEN_CREATE - | OpenFlags::SQLITE_OPEN_NO_MUTEX, - )?; - protect_database_file(path)?; - let database_lock = acquire_database_identity_lock(path)?; - if user_object_count(&conn)? != 0 { - return Err(StoreError::NotAStore); - } +impl AccountDeviceKey { + pub(crate) fn new(account: [u8; 32], device: [u8; 32]) -> Self { + Self { account, device } + } - let mut database_id = [0u8; 32]; - rng.fill_bytes(&mut database_id); - if database_id == [0; 32] { - return Err(StoreError::SchemaMismatch); + pub(crate) fn decode(encoded: &[u8]) -> Result { + if !Self::validate_encoded(encoded) { + return Err(StoreError::LogicalKeyMismatch); } - let metadata = DatabaseMetadata { - database_id, - schema_version: SCHEMA_VERSION, - migrations: vec![MigrationEntry { - id: FOUNDATION_MIGRATION_ID, - from_version: 0, - to_version: SCHEMA_VERSION, - completed: true, - }], - }; - let metadata_key = master.derive(b"KK-store-v2-metadata"); - let sealed_metadata = seal_metadata(&metadata_key, &metadata, rng)?; - - let tx = Transaction::new_unchecked(&conn, TransactionBehavior::Immediate)?; - tx.execute_batch(METADATA_TABLE_SCHEMA)?; - tx.execute_batch(RECORDS_TABLE_SCHEMA)?; - tx.execute_batch(RECORD_LOCATOR_INDEX_SCHEMA)?; - tx.pragma_update(None, "application_id", APPLICATION_ID)?; - tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; - tx.execute( - "INSERT INTO store_v2_metadata (id, blob) VALUES (1, ?1)", - params![sealed_metadata], - )?; - tx.commit()?; - - let target = Self { - conn, - metadata, - index_root: master.derive(b"KK-store-v2-index-root"), - row_root: master.derive(b"KK-store-v2-row-root"), - _database_lock: database_lock, - _lock: lock, - }; - target.validate_complete()?; - Ok(target) + Ok(Self { + account: encoded[1..33] + .try_into() + .map_err(|_| StoreError::LogicalKeyMismatch)?, + device: encoded[33..65] + .try_into() + .map_err(|_| StoreError::LogicalKeyMismatch)?, + }) } - /// Open and fully validate one completed v2 migration destination. - pub fn open(authority: &MigrationAuthority, path: &Path, master: &StorageKey) -> Result { - let () = authority.private; - let lock = acquire_store_lock(path)?; - let conn = Connection::open_with_flags( - path, - OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX, - )?; - let database_lock = acquire_database_identity_lock(path)?; - validate_preamble(&conn)?; - validate_table_names(&conn)?; - let metadata_key = master.derive(b"KK-store-v2-metadata"); - let metadata = read_metadata(&conn, &metadata_key)?; - validate_metadata(&metadata, physical_schema_version(&conn)?)?; - - let target = Self { - conn, - metadata, - index_root: master.derive(b"KK-store-v2-index-root"), - row_root: master.derive(b"KK-store-v2-row-root"), - _database_lock: database_lock, - _lock: lock, - }; - target.validate_complete()?; - Ok(target) + pub(crate) fn account(&self) -> &[u8; 32] { + &self.account } - /// Random identity authenticated inside this database's metadata. - pub fn database_id(&self) -> [u8; 32] { - self.metadata.database_id + pub(crate) fn device(&self) -> &[u8; 32] { + &self.device } +} - /// Explicit physical and authenticated schema version. - pub fn schema_version(&self) -> u32 { - self.metadata.schema_version - } +impl LogicalKey for AccountDeviceKey { + const KIND: u8 = 10; - /// Insert one contact record under a table-separated keyed account index. - pub fn insert_contact( - &self, - key: &AccountKey, - payload: &[u8], - rng: &mut impl CryptoRngCore, - ) -> Result<()> { - self.insert_equality::(key, payload, rng) + fn encode(&self) -> Vec { + let mut encoded = Vec::with_capacity(65); + encoded.push(Self::KIND); + encoded.extend_from_slice(&self.account); + encoded.extend_from_slice(&self.device); + encoded } - /// Read one contact record and recheck its inner account key. - pub fn contact(&self, key: &AccountKey) -> Result>> { - self.get_equality::(key) + fn validate_encoded(encoded: &[u8]) -> bool { + encoded.len() == 65 && encoded.first() == Some(&Self::KIND) } +} - /// Insert one capability record under a domain distinct from contacts. - pub fn insert_capability( - &self, - key: &AccountKey, - payload: &[u8], - rng: &mut impl CryptoRngCore, - ) -> Result<()> { - self.insert_equality::(key, payload, rng) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct MessageDeviceKey { + message: [u8; 16], + device: [u8; 32], +} + +impl MessageDeviceKey { + pub(crate) fn new(message: [u8; 16], device: [u8; 32]) -> Self { + Self { message, device } } - /// Read one capability record and recheck its inner account key. - pub fn capability(&self, key: &AccountKey) -> Result>> { - self.get_equality::(key) + pub(crate) fn decode(encoded: &[u8]) -> Result { + if !Self::validate_encoded(encoded) { + return Err(StoreError::LogicalKeyMismatch); + } + Ok(Self { + message: encoded[1..17] + .try_into() + .map_err(|_| StoreError::LogicalKeyMismatch)?, + device: encoded[17..49] + .try_into() + .map_err(|_| StoreError::LogicalKeyMismatch)?, + }) } - /// Append one message record under a fresh random row locator. - pub fn append_message( - &self, - key: &MessageKey, - payload: &[u8], - rng: &mut impl CryptoRngCore, - ) -> Result { - self.insert_append::(key, payload, rng) + pub(crate) fn message(&self) -> &[u8; 16] { + &self.message } - /// Read an append-only message by opaque locator and expected logical key. - pub fn message( - &self, - locator: &RowLocator, - expected_key: &MessageKey, - ) -> Result>> { - self.get_append::(locator, expected_key) + pub(crate) fn device(&self) -> &[u8; 32] { + &self.device } +} - fn insert_equality( - &self, - key: &T::Key, - payload: &[u8], - rng: &mut impl CryptoRngCore, - ) -> Result<()> { - let locator = self.equality_index::(key); - let plain = encode_record(key, payload)?; - let sealed = self.seal_row(T::DOMAIN, &locator, &plain, rng); - match self.conn.execute( - "INSERT INTO store_v2_records (table_domain, locator, blob) - VALUES (?1, ?2, ?3)", - params![T::DOMAIN as u8, locator.as_slice(), sealed], - ) { - Ok(_) => Ok(()), - Err(error) if constraint_violation(&error) => Err(StoreError::DuplicateIndex), - Err(error) => Err(StoreError::Db(error)), - } +impl LogicalKey for MessageDeviceKey { + const KIND: u8 = 11; + + fn encode(&self) -> Vec { + let mut encoded = Vec::with_capacity(49); + encoded.push(Self::KIND); + encoded.extend_from_slice(&self.message); + encoded.extend_from_slice(&self.device); + encoded } - /// Read one typed equality record and recheck its inner logical key. - fn get_equality(&self, key: &T::Key) -> Result>> { - let locator = self.equality_index::(key); - self.read_exact(T::DOMAIN, &locator, key) + fn validate_encoded(encoded: &[u8]) -> bool { + encoded.len() == 49 && encoded.first() == Some(&Self::KIND) } +} - /// Append one typed record under a fresh random row locator. - fn insert_append( - &self, - key: &T::Key, - payload: &[u8], - rng: &mut impl CryptoRngCore, - ) -> Result { - let plain = encode_record(key, payload)?; - for _ in 0..ROW_LOCATOR_ATTEMPTS { - let mut locator = [0u8; 16]; - rng.fill_bytes(&mut locator); - let sealed = self.seal_row(T::DOMAIN, &locator, &plain, rng); - match self.conn.execute( - "INSERT INTO store_v2_records (table_domain, locator, blob) - VALUES (?1, ?2, ?3)", - params![T::DOMAIN as u8, locator.as_slice(), sealed], - ) { - Ok(_) => return Ok(RowLocator(locator)), - Err(error) if constraint_violation(&error) => continue, - Err(error) => return Err(StoreError::Db(error)), - } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct MetadataKey(Vec); + +impl MetadataKey { + pub(crate) fn new(encoded_key: Vec) -> Result { + if encoded_key.is_empty() || encoded_key.len() > MAX_CANONICAL_KEY_BYTES - 3 { + return Err(StoreError::RecordBounds); } - Err(StoreError::DuplicateIndex) + Ok(Self(encoded_key)) } +} - /// Read an append-only row by its opaque locator and expected logical key. - fn get_append( - &self, - locator: &RowLocator, - expected_key: &T::Key, - ) -> Result>> { - self.read_exact(T::DOMAIN, locator.as_bytes(), expected_key) +impl LogicalKey for MetadataKey { + const KIND: u8 = 12; + + fn encode(&self) -> Vec { + let mut encoded = Vec::with_capacity(3 + self.0.len()); + encoded.push(Self::KIND); + encoded.extend_from_slice(&(self.0.len() as u16).to_be_bytes()); + encoded.extend_from_slice(&self.0); + encoded } - fn read_exact( - &self, - domain: TableDomain, - locator: &[u8], - expected_key: &K, - ) -> Result>> { - let sealed: Option> = self - .conn - .query_row( - "SELECT blob FROM store_v2_records - WHERE table_domain = ?1 AND locator = ?2", - params![domain as u8, locator], - |row| row.get(0), - ) - .optional()?; - let Some(sealed) = sealed else { - return Ok(None); + fn validate_encoded(encoded: &[u8]) -> bool { + if encoded.len() < 4 + || encoded.first() != Some(&Self::KIND) + || encoded.len() > MAX_CANONICAL_KEY_BYTES + { + return false; + } + let Ok(length) = <[u8; 2]>::try_from(&encoded[1..3]) else { + return false; }; - let plain = Zeroizing::new(self.open_row(domain, locator, &sealed)?); - let decoded = decode_record(&plain)?; - validate_logical_key(domain, decoded.logical_key)?; - let expected_key = expected_key.encode(); - if decoded.logical_key != expected_key.as_slice() { + usize::from(u16::from_be_bytes(length)) == encoded.len() - 3 + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct EphemeralKey(Vec); + +impl EphemeralKey { + pub(crate) fn new( + conversation_kind: u8, + conversation: [u8; 32], + author: [u8; 32], + content_id: [u8; 16], + ) -> Result { + if !matches!(conversation_kind, 0 | 1) { return Err(StoreError::LogicalKeyMismatch); } - Ok(Some(decoded.payload.to_vec())) + let mut body = Vec::with_capacity(81); + body.push(conversation_kind); + body.extend_from_slice(&conversation); + body.extend_from_slice(&author); + body.extend_from_slice(&content_id); + Ok(Self(body)) + } +} + +impl LogicalKey for EphemeralKey { + const KIND: u8 = 13; + + fn encode(&self) -> Vec { + let mut encoded = Vec::with_capacity(82); + encoded.push(Self::KIND); + encoded.extend_from_slice(&self.0); + encoded } - fn equality_index(&self, key: &T::Key) -> [u8; 32] { - self.index_for(T::DOMAIN, &key.encode()) + fn validate_encoded(encoded: &[u8]) -> bool { + encoded.len() == 82 + && encoded.first() == Some(&Self::KIND) + && matches!(encoded.get(1), Some(0 | 1)) } +} - fn index_for(&self, domain: TableDomain, logical_key: &[u8]) -> [u8; 32] { - let mut key_label = - Vec::with_capacity(INDEX_KEY_DOMAIN.len() + 32 + core::mem::size_of::()); - key_label.extend_from_slice(INDEX_KEY_DOMAIN); - key_label.extend_from_slice(&self.metadata.database_id); - key_label.push(domain as u8); - let table_key = self.index_root.derive(&key_label); +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct OpaqueRowKey([u8; 16]); - let mut input = Vec::with_capacity(INDEX_INPUT_DOMAIN.len() + logical_key.len()); - input.extend_from_slice(INDEX_INPUT_DOMAIN); - input.extend_from_slice(logical_key); - table_key.hmac_sha256(&input) +impl OpaqueRowKey { + pub(crate) fn from_locator(locator: [u8; 16]) -> Self { + Self(locator) } +} - fn row_key(&self, domain: TableDomain) -> StorageKey { - let mut label = Vec::with_capacity(ROW_KEY_DOMAIN.len() + 32 + 1); - label.extend_from_slice(ROW_KEY_DOMAIN); - label.extend_from_slice(&self.metadata.database_id); - label.push(domain as u8); - self.row_root.derive(&label) +impl LogicalKey for OpaqueRowKey { + const KIND: u8 = 14; + + fn encode(&self) -> Vec { + let mut encoded = Vec::with_capacity(17); + encoded.push(Self::KIND); + encoded.extend_from_slice(&self.0); + encoded } - fn row_ad(&self, domain: TableDomain, locator: &[u8]) -> Vec { - let mut ad = Vec::with_capacity(ROW_AD_DOMAIN.len() + 32 + 4 + 1 + locator.len()); - ad.extend_from_slice(ROW_AD_DOMAIN); - ad.extend_from_slice(&self.metadata.database_id); - ad.extend_from_slice(&self.metadata.schema_version.to_be_bytes()); - ad.push(domain as u8); - ad.extend_from_slice(locator); - ad + fn validate_encoded(encoded: &[u8]) -> bool { + encoded.len() == 17 && encoded.first() == Some(&Self::KIND) } +} - fn seal_row( - &self, - domain: TableDomain, - locator: &[u8], - plain: &[u8], - rng: &mut impl CryptoRngCore, - ) -> Vec { - self.row_key(domain) - .seal(&self.row_ad(domain, locator), plain, rng) +macro_rules! table { + ($name:ident, $domain:expr, $locator:ident, $key:ty) => { + pub(crate) struct $name; + + impl TableSpec for $name { + type Key = $key; + + const DOMAIN: u8 = $domain; + const LOCATOR_KIND: LocatorKind = LocatorKind::$locator; + } + }; +} + +table!(IdentityRows, 1, Equality, SingletonKey); +table!(SessionRows, 2, Equality, AccountKey); +table!(CapabilityRows, 3, Equality, AccountKey); +table!(MessageRows, 4, Random, MessageKey); +table!(QueueRows, 5, Random, OpaqueRowKey); +table!(SeenRows, 6, Equality, ContentKey); +table!(ReceiptReplayRows, 7, Equality, ContentKey); +table!(ContactRows, 8, Equality, AccountKey); +table!(PrekeyRows, 9, Equality, SingletonKey); +table!(PendingRows, 10, Random, OpaqueRowKey); +table!(ResetRows, 11, Equality, AccountKey); +table!(GroupRows, 12, Equality, GroupKey); +table!(GroupAuthorityRows, 13, Equality, GroupKey); +table!(GroupChainRows, 14, Equality, GroupMemberKey); +table!(GroupMessageRows, 15, Random, GroupMessageKey); +table!(MediaTransferRows, 16, Equality, LocalIdKey); +table!(MediaObjectRows, 17, Equality, LocalIdKey); +table!(LocalMetadataRows, 18, Equality, MetadataKey); +table!(NoteRows, 19, Random, ContentKey); +table!(ScheduledRows, 20, Equality, ContentKey); +table!(EphemeralRows, 21, Equality, EphemeralKey); +table!(DeviceStateRows, 22, Equality, SingletonKey); +table!(DeviceSyncRows, 23, Random, DigestKey); +table!(ContactDeviceRows, 24, Equality, AccountDeviceKey); +table!(MessageDeviceDeliveryRows, 25, Equality, MessageDeviceKey); +pub(crate) struct MigrationCheckpointRows; + +impl TableSpec for MigrationCheckpointRows { + type Key = SingletonKey; + + const DOMAIN: u8 = 250; + const LOCATOR_KIND: LocatorKind = LocatorKind::Equality; +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct IndexKeys { + unique: Option>, + a: Option>, + b: Option>, + c: Option>, + d: Option>, +} + +impl IndexKeys { + pub(crate) fn none() -> Self { + Self::default() } - fn open_row(&self, domain: TableDomain, locator: &[u8], sealed: &[u8]) -> Result> { - Ok(self - .row_key(domain) - .open(&self.row_ad(domain, locator), sealed)?) + pub(crate) fn message(id: &ContentKey, conversation: &AccountKey) -> Self { + Self { + unique: Some(id.encode()), + a: Some(conversation.encode()), + ..Self::default() + } } - fn validate_complete(&self) -> Result<()> { - validate_preamble(&self.conn)?; - validate_table_names(&self.conn)?; - validate_metadata(&self.metadata, physical_schema_version(&self.conn)?)?; - validate_duplicate_locators(&self.conn)?; - validate_columns_and_indexes(&self.conn)?; - self.validate_rows() + pub(crate) fn group_message(id: &ContentKey, conversation: &GroupKey) -> Self { + Self { + unique: Some(id.encode()), + a: Some(conversation.encode()), + ..Self::default() + } } - fn validate_rows(&self) -> Result<()> { - let mut statement = self.conn.prepare( - "SELECT table_domain, locator, blob - FROM store_v2_records ORDER BY rowid_", - )?; - let rows = statement.query_map([], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, Vec>(1)?, - row.get::<_, Vec>(2)?, - )) - })?; - for row in rows { - let (domain, locator, sealed) = row?; - let domain = TableDomain::from_sql(domain)?; - validate_locator(domain, &locator)?; - let plain = Zeroizing::new(self.open_row(domain, &locator, &sealed)?); - let decoded = decode_record(&plain)?; - validate_logical_key(domain, decoded.logical_key)?; - if domain.locator_kind() == LocatorKind::Equality - && self.index_for(domain, decoded.logical_key).as_slice() != locator.as_slice() - { - return Err(StoreError::LogicalKeyMismatch); - } + pub(crate) fn queue( + peer: &AccountKey, + message: Option<&ContentKey>, + group_message: Option<&ContentKey>, + envelope: &ContentKey, + ) -> Self { + Self { + a: Some(peer.encode()), + b: message.map(LogicalKey::encode), + c: group_message.map(LogicalKey::encode), + d: Some(envelope.encode()), + ..Self::default() } - Ok(()) } -} -pub(crate) fn is_inactive_migration_target(conn: &Connection) -> Result { - let application_id: u32 = conn.pragma_query_value(None, "application_id", |row| row.get(0))?; + pub(crate) fn group_chain(group: &GroupKey) -> Self { + Self { + a: Some(group.encode()), + ..Self::default() + } + } + + pub(crate) fn media_object(transfer: &LocalIdKey) -> Self { + Self { + a: Some(transfer.encode()), + ..Self::default() + } + } + + pub(crate) fn note(id: &ContentKey) -> Self { + Self { + unique: Some(id.encode()), + ..Self::default() + } + } + + pub(crate) fn device_sync(digest: &DigestKey) -> Self { + Self { + unique: Some(digest.encode()), + ..Self::default() + } + } + + pub(crate) fn contact_device(account: &AccountKey) -> Self { + Self { + a: Some(account.encode()), + ..Self::default() + } + } + + pub(crate) fn message_device_delivery(message: &ContentKey, account: &AccountKey) -> Self { + Self { + a: Some(message.encode()), + b: Some(account.encode()), + ..Self::default() + } + } + + fn as_array(&self) -> [Option<&[u8]>; 5] { + [ + self.unique.as_deref(), + self.a.as_deref(), + self.b.as_deref(), + self.c.as_deref(), + self.d.as_deref(), + ] + } +} + +pub(crate) trait LookupIndex { + type Table: TableSpec; + type Key: LogicalKey; + + const SLOT: usize; + const COLUMN: &'static str; +} + +macro_rules! lookup_index { + ($name:ident, $table:ty, $key:ty, $slot:expr, $column:literal) => { + pub(crate) struct $name; + + impl LookupIndex for $name { + type Table = $table; + type Key = $key; + + const SLOT: usize = $slot; + const COLUMN: &'static str = $column; + } + }; +} + +lookup_index!(MessageIdIndex, MessageRows, ContentKey, 1, "unique_index"); +lookup_index!( + MessageConversationIndex, + MessageRows, + AccountKey, + 2, + "index_a" +); +lookup_index!(QueuePeerIndex, QueueRows, AccountKey, 2, "index_a"); +lookup_index!(QueueMessageIndex, QueueRows, ContentKey, 3, "index_b"); +lookup_index!(QueueGroupMessageIndex, QueueRows, ContentKey, 4, "index_c"); +lookup_index!(QueueEnvelopeIndex, QueueRows, ContentKey, 5, "index_d"); +lookup_index!(GroupChainGroupIndex, GroupChainRows, GroupKey, 2, "index_a"); +lookup_index!( + GroupMessageIdIndex, + GroupMessageRows, + ContentKey, + 1, + "unique_index" +); +lookup_index!( + GroupMessageConversationIndex, + GroupMessageRows, + GroupKey, + 2, + "index_a" +); +lookup_index!( + MediaObjectTransferIndex, + MediaObjectRows, + LocalIdKey, + 2, + "index_a" +); +lookup_index!( + DeviceSyncDigestIndex, + DeviceSyncRows, + DigestKey, + 1, + "unique_index" +); +lookup_index!( + ContactDeviceAccountIndex, + ContactDeviceRows, + AccountKey, + 2, + "index_a" +); +lookup_index!( + MessageDeliveryMessageIndex, + MessageDeviceDeliveryRows, + ContentKey, + 2, + "index_a" +); +lookup_index!( + MessageDeliveryAccountIndex, + MessageDeviceDeliveryRows, + AccountKey, + 3, + "index_b" +); + +#[derive(Debug)] +pub(crate) struct RawRow { + pub(crate) rowid: i64, + pub(crate) locator: Vec, + pub(crate) logical_key: Vec, + indexes: IndexKeys, + pub(crate) payload: Zeroizing>, +} + +impl RawRow { + pub(crate) fn verify_key(&self, expected: &K) -> Result<()> { + if self.logical_key == expected.encode() { + Ok(()) + } else { + Err(StoreError::LogicalKeyMismatch) + } + } + + pub(crate) fn verify_indexes(&self, expected: &IndexKeys) -> Result<()> { + if &self.indexes == expected { + Ok(()) + } else { + Err(StoreError::LogicalKeyMismatch) + } + } +} + +struct DecodedRecord<'a> { + logical_key: &'a [u8], + indexes: [Option<&'a [u8]>; 5], + payload: &'a [u8], +} + +pub(crate) struct DerivedKeys { + pub(crate) index_root: StorageKey, + pub(crate) row_root: StorageKey, + pub(crate) cursor_root: StorageKey, + pub(crate) media: StorageKey, +} + +pub(crate) fn derive_store_keys(master: &StorageKey) -> DerivedKeys { + DerivedKeys { + index_root: master.derive(b"KK-store-v2-index-root"), + row_root: master.derive(b"KK-store-v2-row-root"), + cursor_root: master.derive(CURSOR_KEY_DOMAIN), + media: master.derive(b"KK-store-media"), + } +} + +pub(crate) fn random_database_id(rng: &mut impl CryptoRngCore) -> Result<[u8; 32]> { + let mut database_id = [0u8; 32]; + rng.fill_bytes(&mut database_id); + if database_id == [0; 32] { + return Err(StoreError::SchemaMismatch); + } + Ok(database_id) +} + +pub(crate) fn create_schema(conn: &Connection) -> Result<()> { + conn.pragma_update(None, "auto_vacuum", "INCREMENTAL")?; + conn.execute_batch(BOOTSTRAP_TABLE_SCHEMA)?; + conn.execute_batch(METADATA_TABLE_SCHEMA)?; + conn.execute_batch(RECORDS_TABLE_SCHEMA)?; + conn.execute_batch(RECORD_LOCATOR_INDEX_SCHEMA)?; + conn.execute_batch(RECORD_UNIQUE_INDEX_SCHEMA)?; + conn.execute_batch(RECORD_INDEX_A_SCHEMA)?; + conn.execute_batch(RECORD_INDEX_B_SCHEMA)?; + conn.execute_batch(RECORD_INDEX_C_SCHEMA)?; + conn.execute_batch(RECORD_INDEX_D_SCHEMA)?; + conn.pragma_update(None, "application_id", APPLICATION_ID)?; + conn.pragma_update(None, "user_version", SCHEMA_VERSION)?; + Ok(()) +} + +pub(crate) fn configure_connection(conn: &Connection) -> Result<()> { + conn.pragma_update(None, "foreign_keys", true)?; + conn.pragma_update(None, "secure_delete", true)?; + conn.pragma_update(None, "synchronous", "FULL")?; + conn.pragma_update(None, "wal_autocheckpoint", 1_000u32)?; + conn.busy_timeout(std::time::Duration::from_secs(5))?; + Ok(()) +} + +pub(crate) fn write_bootstrap( + conn: &Connection, + salt: &[u8; 16], + profile: KdfProfile, + wrapped_sk: &[u8], +) -> Result<()> { + let kdf = postcard::to_allocvec(&(profile.m_cost_kib, profile.t_cost, profile.p_cost)) + .map_err(|_| StoreError::Serialization)?; + conn.execute( + "INSERT INTO store_bootstrap (id, salt, kdf, wrapped_sk) VALUES (1, ?1, ?2, ?3)", + params![salt.as_slice(), kdf, wrapped_sk], + )?; + Ok(()) +} + +pub(crate) fn read_bootstrap(conn: &Connection) -> Result<([u8; 16], KdfProfile, Vec)> { + let count: i64 = + conn.query_row("SELECT COUNT(*) FROM store_bootstrap", [], |row| row.get(0))?; + if count != 1 { + return Err(StoreError::SchemaMismatch); + } + let (salt, kdf, wrapped): (Vec, Vec, Vec) = conn + .query_row( + "SELECT salt, kdf, wrapped_sk FROM store_bootstrap WHERE id = 1", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()? + .ok_or(StoreError::NotAStore)?; + let salt: [u8; 16] = salt.try_into().map_err(|_| StoreError::NotAStore)?; + let (m_cost_kib, t_cost, p_cost): (u32, u32, u32) = + decode_exact(&kdf).map_err(|_| StoreError::NotAStore)?; + if wrapped.len() < 24 + 16 { + return Err(StoreError::NotAStore); + } + Ok(( + salt, + KdfProfile { + m_cost_kib, + t_cost, + p_cost, + }, + wrapped, + )) +} + +pub(crate) fn write_metadata( + conn: &Connection, + master: &StorageKey, + metadata: &DatabaseMetadata, + rng: &mut impl CryptoRngCore, +) -> Result<()> { + let metadata_key = master.derive(b"KK-store-v2-metadata"); + write_metadata_with_key(conn, &metadata_key, metadata, rng) +} + +pub(crate) fn write_metadata_with_key( + conn: &Connection, + metadata_key: &StorageKey, + metadata: &DatabaseMetadata, + rng: &mut impl CryptoRngCore, +) -> Result<()> { + let sealed = seal_metadata(metadata_key, metadata, rng)?; + conn.execute( + "INSERT INTO store_metadata (id, blob) VALUES (1, ?1) + ON CONFLICT(id) DO UPDATE SET blob = excluded.blob", + params![sealed], + )?; + Ok(()) +} + +pub(crate) fn read_metadata( + conn: &Connection, + master: &StorageKey, + allow_incomplete: bool, +) -> Result { + let count: i64 = conn.query_row("SELECT COUNT(*) FROM store_metadata", [], |row| row.get(0))?; + if count != 1 { + return Err(StoreError::SchemaMismatch); + } + let sealed: Vec = conn + .query_row("SELECT blob FROM store_metadata WHERE id = 1", [], |row| { + row.get(0) + }) + .optional()? + .ok_or(StoreError::SchemaMismatch)?; + let metadata_key = master.derive(b"KK-store-v2-metadata"); + let metadata = open_metadata(&metadata_key, &sealed)?; + validate_metadata(&metadata, physical_schema_version(conn)?, allow_incomplete)?; + Ok(metadata) +} + +pub(crate) fn validate_physical_schema(conn: &Connection) -> Result<()> { + validate_preamble(conn)?; + let mut statement = conn.prepare( + "SELECT type, name, sql FROM sqlite_schema + WHERE name NOT LIKE 'sqlite_%' ORDER BY name", + )?; + let objects = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + })? + .collect::>>()?; + let mut expected = vec![ + ( + "table".to_owned(), + "store_bootstrap".to_owned(), + BOOTSTRAP_TABLE_SCHEMA.to_owned(), + ), + ( + "index".to_owned(), + "store_record_index_a".to_owned(), + RECORD_INDEX_A_SCHEMA.to_owned(), + ), + ( + "index".to_owned(), + "store_record_index_b".to_owned(), + RECORD_INDEX_B_SCHEMA.to_owned(), + ), + ( + "index".to_owned(), + "store_record_index_c".to_owned(), + RECORD_INDEX_C_SCHEMA.to_owned(), + ), + ( + "index".to_owned(), + "store_record_index_d".to_owned(), + RECORD_INDEX_D_SCHEMA.to_owned(), + ), + ( + "index".to_owned(), + "store_record_locator".to_owned(), + RECORD_LOCATOR_INDEX_SCHEMA.to_owned(), + ), + ( + "index".to_owned(), + "store_record_unique".to_owned(), + RECORD_UNIQUE_INDEX_SCHEMA.to_owned(), + ), + ( + "table".to_owned(), + "store_records".to_owned(), + RECORDS_TABLE_SCHEMA.to_owned(), + ), + ( + "table".to_owned(), + "store_metadata".to_owned(), + METADATA_TABLE_SCHEMA.to_owned(), + ), + ]; + expected.sort_by(|left, right| left.1.cmp(&right.1)); + if objects != expected { + return Err(StoreError::SchemaMismatch); + } + Ok(()) +} + +pub(crate) fn is_v2(conn: &Connection) -> Result { + let application_id: u32 = conn.pragma_query_value(None, "application_id", |row| row.get(0))?; if application_id == APPLICATION_ID { return Ok(true); } let found: Option = conn .query_row( "SELECT 1 FROM sqlite_schema - WHERE name IN ('store_v2_metadata', 'store_v2_records') + WHERE name IN ('store_bootstrap', 'store_metadata', 'store_records') LIMIT 1", [], |row| row.get(0), @@ -603,12 +1032,750 @@ pub(crate) fn is_inactive_migration_target(conn: &Connection) -> Result { Ok(found.is_some()) } -fn encode_record(key: &K, payload: &[u8]) -> Result> { - let logical_key = key.encode(); - let key_len = u16::try_from(logical_key.len()).map_err(|_| StoreError::RecordBounds)?; +pub(crate) fn physical_schema_version(conn: &Connection) -> Result { + Ok(conn.pragma_query_value(None, "user_version", |row| row.get(0))?) +} + +impl Store { + pub(crate) fn put_equality( + &self, + key: &T::Key, + payload: &[u8], + indexes: IndexKeys, + rng: &mut impl CryptoRngCore, + ) -> Result<()> { + self.put_equality_on::(&self.conn, key, payload, indexes, rng) + } + + pub(crate) fn put_equality_on( + &self, + conn: &Connection, + key: &T::Key, + payload: &[u8], + indexes: IndexKeys, + rng: &mut impl CryptoRngCore, + ) -> Result<()> { + if T::LOCATOR_KIND != LocatorKind::Equality { + return Err(StoreError::SchemaMismatch); + } + validate_index_shape(T::DOMAIN, &indexes)?; + let logical_key = key.encode(); + let locator = self.index_for(T::DOMAIN, 0, &logical_key); + let sealed = self.seal_logical_row(T::DOMAIN, &locator, key, &indexes, payload, rng)?; + let physical = self.physical_indexes(T::DOMAIN, &indexes); + match conn.execute( + "INSERT INTO store_records + (table_domain, locator, unique_index, index_a, index_b, index_c, index_d, blob) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(table_domain, locator) DO UPDATE SET + unique_index = excluded.unique_index, + index_a = excluded.index_a, + index_b = excluded.index_b, + index_c = excluded.index_c, + index_d = excluded.index_d, + blob = excluded.blob", + params![ + T::DOMAIN, + locator.as_slice(), + physical[0].as_deref(), + physical[1].as_deref(), + physical[2].as_deref(), + physical[3].as_deref(), + physical[4].as_deref(), + sealed + ], + ) { + Ok(_) => Ok(()), + Err(error) if constraint_violation(&error) => Err(StoreError::DuplicateIndex), + Err(error) => Err(StoreError::Db(error)), + } + } + + pub(crate) fn append( + &self, + key: &T::Key, + payload: &[u8], + indexes: IndexKeys, + rng: &mut impl CryptoRngCore, + ) -> Result { + self.append_on::(&self.conn, Some(key), payload, indexes, rng) + } + + pub(crate) fn append_opaque( + &self, + payload: &[u8], + indexes: IndexKeys, + rng: &mut impl CryptoRngCore, + ) -> Result + where + T: TableSpec, + { + self.append_on::(&self.conn, None, payload, indexes, rng) + } + + pub(crate) fn append_on( + &self, + conn: &Connection, + key: Option<&T::Key>, + payload: &[u8], + indexes: IndexKeys, + rng: &mut impl CryptoRngCore, + ) -> Result { + if T::LOCATOR_KIND != LocatorKind::Random { + return Err(StoreError::SchemaMismatch); + } + validate_index_shape(T::DOMAIN, &indexes)?; + for _ in 0..ROW_LOCATOR_ATTEMPTS { + let mut locator = [0u8; 16]; + rng.fill_bytes(&mut locator); + let generated; + let key = match key { + Some(key) => key, + None if T::Key::KIND == OpaqueRowKey::KIND => { + generated = OpaqueRowKey::from_locator(locator); + // This branch is reachable only through `append_opaque`, + // whose associated key type is statically constrained. + let encoded = generated.encode(); + let sealed = self.seal_encoded_row( + T::DOMAIN, + &locator, + &encoded, + &indexes, + payload, + rng, + )?; + let physical = self.physical_indexes(T::DOMAIN, &indexes); + match insert_record(conn, T::DOMAIN, &locator, &physical, &sealed) { + Ok(rowid) => { + return Ok(RawRow { + rowid, + locator: locator.to_vec(), + logical_key: encoded, + indexes: indexes.clone(), + payload: Zeroizing::new(payload.to_vec()), + }); + } + Err(StoreError::DuplicateIndex) => continue, + Err(error) => return Err(error), + } + } + None => return Err(StoreError::LogicalKeyMismatch), + }; + let encoded = key.encode(); + let sealed = + self.seal_encoded_row(T::DOMAIN, &locator, &encoded, &indexes, payload, rng)?; + let physical = self.physical_indexes(T::DOMAIN, &indexes); + match insert_record(conn, T::DOMAIN, &locator, &physical, &sealed) { + Ok(rowid) => { + return Ok(RawRow { + rowid, + locator: locator.to_vec(), + logical_key: encoded, + indexes: indexes.clone(), + payload: Zeroizing::new(payload.to_vec()), + }); + } + Err(StoreError::DuplicateIndex) => continue, + Err(error) => return Err(error), + } + } + Err(StoreError::DuplicateIndex) + } + + pub(crate) fn get_equality(&self, key: &T::Key) -> Result> { + if T::LOCATOR_KIND != LocatorKind::Equality { + return Err(StoreError::SchemaMismatch); + } + let encoded = key.encode(); + let locator = self.index_for(T::DOMAIN, 0, &encoded); + self.raw_row_by_locator::(&self.conn, &locator)? + .map(|row| { + row.verify_key(key)?; + Ok(row) + }) + .transpose() + } + + pub(crate) fn delete_equality(&self, key: &T::Key) -> Result { + self.delete_equality_on::(&self.conn, key) + } + + pub(crate) fn delete_equality_on( + &self, + conn: &Connection, + key: &T::Key, + ) -> Result { + if T::LOCATOR_KIND != LocatorKind::Equality { + return Err(StoreError::SchemaMismatch); + } + let locator = self.index_for(T::DOMAIN, 0, &key.encode()); + Ok(conn.execute( + "DELETE FROM store_records WHERE table_domain = ?1 AND locator = ?2", + params![T::DOMAIN, locator.as_slice()], + )? == 1) + } + + pub(crate) fn rows(&self) -> Result> { + self.rows_on::(&self.conn) + } + + pub(crate) fn row_by_rowid(&self, rowid: i64) -> Result> { + self.row_by_rowid_on::(&self.conn, rowid) + } + + pub(crate) fn row_by_rowid_on( + &self, + conn: &Connection, + rowid: i64, + ) -> Result> { + let row = conn + .query_row( + "SELECT rowid_, locator, unique_index, index_a, index_b, index_c, index_d, blob + FROM store_records WHERE table_domain = ?1 AND rowid_ = ?2", + params![T::DOMAIN, rowid], + sql_row, + ) + .optional()?; + row.map(|row| self.open_sql_row::(row)).transpose() + } + + pub(crate) fn rows_on(&self, conn: &Connection) -> Result> { + let mut statement = conn.prepare( + "SELECT rowid_, locator, unique_index, index_a, index_b, index_c, index_d, blob + FROM store_records WHERE table_domain = ?1 ORDER BY rowid_", + )?; + let rows = statement.query_map(params![T::DOMAIN], sql_row)?; + let mut decoded = Vec::new(); + for row in rows { + decoded.push(self.open_sql_row::(row?)?); + } + Ok(decoded) + } + + pub(crate) fn validate_rows(&self, mut validate: F) -> Result<()> + where + T: TableSpec, + F: FnMut(&RawRow) -> Result<()>, + { + let mut statement = self.conn.prepare( + "SELECT rowid_, locator, unique_index, index_a, index_b, index_c, index_d, blob + FROM store_records WHERE table_domain = ?1 ORDER BY rowid_", + )?; + let rows = statement.query_map(params![T::DOMAIN], sql_row)?; + for row in rows { + let decoded = self.open_sql_row::(row?)?; + validate(&decoded)?; + } + Ok(()) + } + + pub(crate) fn row_by_unique(&self, key: &I::Key) -> Result> { + let index = self.index_for( + ::DOMAIN, + I::SLOT as u8, + &key.encode(), + ); + let sql = format!( + "SELECT rowid_, locator, unique_index, index_a, index_b, index_c, index_d, blob + FROM store_records WHERE table_domain = ?1 AND {} = ?2", + I::COLUMN + ); + let row = self + .conn + .query_row( + &sql, + params![::DOMAIN, index.as_slice()], + sql_row, + ) + .optional()?; + row.map(|row| self.open_sql_row::(row)) + .transpose() + } + + pub(crate) fn rows_by_index(&self, key: &I::Key) -> Result> { + self.rows_by_index_after::(key, None, i64::MAX as usize) + } + + pub(crate) fn rows_by_index_after( + &self, + key: &I::Key, + after_rowid: Option, + limit: usize, + ) -> Result> { + if limit == 0 { + return Ok(Vec::new()); + } + let limit = i64::try_from(limit).map_err(|_| StoreError::RecordBounds)?; + let index = self.index_for( + ::DOMAIN, + I::SLOT as u8, + &key.encode(), + ); + let sql = format!( + "SELECT rowid_, locator, unique_index, index_a, index_b, index_c, index_d, blob + FROM store_records + WHERE table_domain = ?1 AND {} = ?2 AND rowid_ > ?3 + ORDER BY rowid_ LIMIT ?4", + I::COLUMN + ); + let mut statement = self.conn.prepare(&sql)?; + let rows = statement.query_map( + params![ + ::DOMAIN, + index.as_slice(), + after_rowid.unwrap_or(0), + limit + ], + sql_row, + )?; + let mut decoded = Vec::new(); + for row in rows { + decoded.push(self.open_sql_row::(row?)?); + } + Ok(decoded) + } + + pub(crate) fn update_row( + &self, + locator: &[u8], + expected_key: &T::Key, + payload: &[u8], + indexes: IndexKeys, + rng: &mut impl CryptoRngCore, + ) -> Result { + self.update_row_on::(&self.conn, locator, expected_key, payload, indexes, rng) + } + + pub(crate) fn update_row_on( + &self, + conn: &Connection, + locator: &[u8], + expected_key: &T::Key, + payload: &[u8], + indexes: IndexKeys, + rng: &mut impl CryptoRngCore, + ) -> Result { + let Some(current) = self.raw_row_by_locator::(conn, locator)? else { + return Ok(false); + }; + current.verify_key(expected_key)?; + validate_index_shape(T::DOMAIN, &indexes)?; + let sealed = + self.seal_logical_row(T::DOMAIN, locator, expected_key, &indexes, payload, rng)?; + let physical = self.physical_indexes(T::DOMAIN, &indexes); + match conn.execute( + "UPDATE store_records SET unique_index = ?1, index_a = ?2, index_b = ?3, + index_c = ?4, index_d = ?5, blob = ?6 + WHERE table_domain = ?7 AND locator = ?8", + params![ + physical[0].as_deref(), + physical[1].as_deref(), + physical[2].as_deref(), + physical[3].as_deref(), + physical[4].as_deref(), + sealed, + T::DOMAIN, + locator + ], + ) { + Ok(changed) => Ok(changed == 1), + Err(error) if constraint_violation(&error) => Err(StoreError::DuplicateIndex), + Err(error) => Err(StoreError::Db(error)), + } + } + + pub(crate) fn delete_row(&self, locator: &[u8]) -> Result { + Ok(self.conn.execute( + "DELETE FROM store_records WHERE table_domain = ?1 AND locator = ?2", + params![T::DOMAIN, locator], + )? == 1) + } + + pub(crate) fn delete_rowid(&self, rowid: i64) -> Result { + self.delete_rowid_on::(&self.conn, rowid) + } + + pub(crate) fn delete_rowid_on( + &self, + conn: &Connection, + rowid: i64, + ) -> Result { + Ok(conn.execute( + "DELETE FROM store_records WHERE table_domain = ?1 AND rowid_ = ?2", + params![T::DOMAIN, rowid], + )? == 1) + } + + pub(crate) fn delete_row_on( + &self, + conn: &Connection, + locator: &[u8], + ) -> Result { + Ok(conn.execute( + "DELETE FROM store_records WHERE table_domain = ?1 AND locator = ?2", + params![T::DOMAIN, locator], + )? == 1) + } + + pub(crate) fn count_rows(&self) -> Result { + self.count_rows_on::(&self.conn) + } + + pub(crate) fn count_rows_on(&self, conn: &Connection) -> Result { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM store_records WHERE table_domain = ?1", + params![T::DOMAIN], + |row| row.get(0), + )?; + u64::try_from(count).map_err(|_| StoreError::Serialization) + } + + pub(crate) fn sealed_bytes_on(&self, conn: &Connection) -> Result { + let bytes: i64 = conn.query_row( + "SELECT COALESCE(SUM(length(blob)), 0) FROM store_records WHERE table_domain = ?1", + params![T::DOMAIN], + |row| row.get(0), + )?; + u64::try_from(bytes).map_err(|_| StoreError::Serialization) + } + + pub(crate) fn validate_all_opaque_rows(&self) -> Result<()> { + let mut statement = self.conn.prepare( + "SELECT rowid_, table_domain, locator, unique_index, index_a, index_b, index_c, + index_d, blob + FROM store_records ORDER BY rowid_", + )?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, Vec>(2)?, + row.get::<_, Option>>(3)?, + row.get::<_, Option>>(4)?, + row.get::<_, Option>>(5)?, + row.get::<_, Option>>(6)?, + row.get::<_, Option>>(7)?, + row.get::<_, Vec>(8)?, + )) + })?; + for row in rows { + let (_, domain, locator, unique, a, b, c, d, sealed) = row?; + let domain = u8::try_from(domain).map_err(|_| StoreError::SchemaMismatch)?; + let locator_kind = table_locator_kind(domain)?; + let expected_locator_len = match locator_kind { + LocatorKind::Equality => 32, + LocatorKind::Random => 16, + }; + if locator.len() != expected_locator_len { + return Err(StoreError::SchemaMismatch); + } + let physical = [unique, a, b, c, d]; + let plain = Zeroizing::new(self.open_row(domain, &locator, &sealed)?); + let decoded = decode_record(&plain)?; + validate_key_for_domain(domain, decoded.logical_key)?; + let indexes = decoded_indexes_owned(&decoded)?; + validate_index_shape(domain, &indexes)?; + let expected = self.physical_indexes(domain, &indexes); + if physical != expected { + return Err(StoreError::LogicalKeyMismatch); + } + if locator_kind == LocatorKind::Equality + && self.index_for(domain, 0, decoded.logical_key).as_slice() != locator.as_slice() + { + return Err(StoreError::LogicalKeyMismatch); + } + if locator_kind == LocatorKind::Random + && domain_uses_opaque_key(domain) + && decoded.logical_key.get(1..) != Some(locator.as_slice()) + { + return Err(StoreError::LogicalKeyMismatch); + } + } + Ok(()) + } + + pub(crate) fn encode_cursor(&self, key: &I::Key, row: &RawRow) -> Vec { + let index = self.index_for( + ::DOMAIN, + I::SLOT as u8, + &key.encode(), + ); + let mut mask_input = Vec::with_capacity(CURSOR_MASK_DOMAIN.len() + 1 + 32 + 16); + mask_input.extend_from_slice(CURSOR_MASK_DOMAIN); + mask_input.push(::DOMAIN); + mask_input.extend_from_slice(&index); + mask_input.extend_from_slice(&row.locator); + let mask = self.cursor_root.hmac_sha256(&mask_input); + let order = row.rowid.to_be_bytes(); + let mut masked = [0u8; 8]; + for (out, (value, key_byte)) in masked.iter_mut().zip(order.iter().zip(mask.iter())) { + *out = value ^ key_byte; + } + let mut token = Vec::with_capacity(1 + 16 + 8 + 32); + token.push(1); + token.extend_from_slice(&row.locator); + token.extend_from_slice(&masked); + let mut tag_input = Vec::with_capacity(CURSOR_TAG_DOMAIN.len() + 1 + 32 + token.len()); + tag_input.extend_from_slice(CURSOR_TAG_DOMAIN); + tag_input.push(::DOMAIN); + tag_input.extend_from_slice(&index); + tag_input.extend_from_slice(&token); + token.extend_from_slice(&self.cursor_root.hmac_sha256(&tag_input)); + token + } + + pub(crate) fn decode_cursor(&self, key: &I::Key, token: &[u8]) -> Result { + if token.len() != 57 || token.first() != Some(&1) { + return Err(StoreError::InvalidCursor); + } + let index = self.index_for( + ::DOMAIN, + I::SLOT as u8, + &key.encode(), + ); + let mut tag_input = Vec::with_capacity(CURSOR_TAG_DOMAIN.len() + 1 + 32 + 25); + tag_input.extend_from_slice(CURSOR_TAG_DOMAIN); + tag_input.push(::DOMAIN); + tag_input.extend_from_slice(&index); + tag_input.extend_from_slice(&token[..25]); + if self.cursor_root.hmac_sha256(&tag_input).as_slice() != &token[25..] { + return Err(StoreError::InvalidCursor); + } + let locator = &token[1..17]; + let mut mask_input = Vec::with_capacity(CURSOR_MASK_DOMAIN.len() + 1 + 32 + 16); + mask_input.extend_from_slice(CURSOR_MASK_DOMAIN); + mask_input.push(::DOMAIN); + mask_input.extend_from_slice(&index); + mask_input.extend_from_slice(locator); + let mask = self.cursor_root.hmac_sha256(&mask_input); + let mut order = [0u8; 8]; + for (out, (value, key_byte)) in order.iter_mut().zip(token[17..25].iter().zip(mask.iter())) + { + *out = value ^ key_byte; + } + let rowid = i64::from_be_bytes(order); + if rowid <= 0 { + return Err(StoreError::InvalidCursor); + } + let exists: Option = self + .conn + .query_row( + "SELECT 1 FROM store_records WHERE table_domain = ?1 AND rowid_ = ?2 + AND locator = ?3 LIMIT 1", + params![::DOMAIN, rowid, locator], + |row| row.get(0), + ) + .optional()?; + if exists.is_none() { + return Err(StoreError::InvalidCursor); + } + Ok(rowid) + } + + fn raw_row_by_locator( + &self, + conn: &Connection, + locator: &[u8], + ) -> Result> { + let row = conn + .query_row( + "SELECT rowid_, locator, unique_index, index_a, index_b, index_c, index_d, blob + FROM store_records WHERE table_domain = ?1 AND locator = ?2", + params![T::DOMAIN, locator], + sql_row, + ) + .optional()?; + row.map(|row| self.open_sql_row::(row)).transpose() + } + + fn open_sql_row(&self, row: SqlRow) -> Result { + let physical = [ + row.unique_index, + row.index_a, + row.index_b, + row.index_c, + row.index_d, + ]; + let plain = Zeroizing::new(self.open_row(T::DOMAIN, &row.locator, &row.blob)?); + let decoded = decode_record(&plain)?; + if !T::Key::validate_encoded(decoded.logical_key) { + return Err(StoreError::LogicalKeyMismatch); + } + let indexes = decoded_indexes_owned(&decoded)?; + validate_index_shape(T::DOMAIN, &indexes)?; + if self.physical_indexes(T::DOMAIN, &indexes) != physical { + return Err(StoreError::LogicalKeyMismatch); + } + if T::LOCATOR_KIND == LocatorKind::Equality + && self.index_for(T::DOMAIN, 0, decoded.logical_key).as_slice() + != row.locator.as_slice() + { + return Err(StoreError::LogicalKeyMismatch); + } + if T::LOCATOR_KIND == LocatorKind::Random + && T::Key::KIND == OpaqueRowKey::KIND + && decoded.logical_key.get(1..) != Some(row.locator.as_slice()) + { + return Err(StoreError::LogicalKeyMismatch); + } + Ok(RawRow { + rowid: row.rowid, + locator: row.locator, + logical_key: decoded.logical_key.to_vec(), + indexes, + payload: Zeroizing::new(decoded.payload.to_vec()), + }) + } + + fn seal_logical_row( + &self, + domain: u8, + locator: &[u8], + key: &K, + indexes: &IndexKeys, + payload: &[u8], + rng: &mut impl CryptoRngCore, + ) -> Result> { + self.seal_encoded_row(domain, locator, &key.encode(), indexes, payload, rng) + } + + fn seal_encoded_row( + &self, + domain: u8, + locator: &[u8], + logical_key: &[u8], + indexes: &IndexKeys, + payload: &[u8], + rng: &mut impl CryptoRngCore, + ) -> Result> { + let plain = Zeroizing::new(encode_record(logical_key, indexes, payload)?); + Ok(self + .row_key(domain) + .seal(&self.row_ad(domain, locator), &plain, rng)) + } + + fn open_row(&self, domain: u8, locator: &[u8], sealed: &[u8]) -> Result> { + Ok(self + .row_key(domain) + .open(&self.row_ad(domain, locator), sealed)?) + } + + fn physical_indexes(&self, domain: u8, indexes: &IndexKeys) -> [Option>; 5] { + let keys = indexes.as_array(); + std::array::from_fn(|position| { + keys[position].map(|key| self.index_for(domain, (position + 1) as u8, key).to_vec()) + }) + } + + fn index_for(&self, table_domain: u8, index_domain: u8, logical_key: &[u8]) -> [u8; 32] { + let mut key_label = Vec::with_capacity(INDEX_KEY_DOMAIN.len() + 32 + 2); + key_label.extend_from_slice(INDEX_KEY_DOMAIN); + key_label.extend_from_slice(&self.metadata.database_id); + key_label.push(table_domain); + key_label.push(index_domain); + let index_key = self.index_root.derive(&key_label); + + let mut input = Vec::with_capacity(INDEX_INPUT_DOMAIN.len() + logical_key.len()); + input.extend_from_slice(INDEX_INPUT_DOMAIN); + input.extend_from_slice(logical_key); + index_key.hmac_sha256(&input) + } + + fn row_key(&self, table_domain: u8) -> StorageKey { + let mut label = Vec::with_capacity(ROW_KEY_DOMAIN.len() + 32 + 1); + label.extend_from_slice(ROW_KEY_DOMAIN); + label.extend_from_slice(&self.metadata.database_id); + label.push(table_domain); + self.row_root.derive(&label) + } + + fn row_ad(&self, table_domain: u8, locator: &[u8]) -> Vec { + let mut ad = Vec::with_capacity(ROW_AD_DOMAIN.len() + 32 + 4 + 1 + locator.len()); + ad.extend_from_slice(ROW_AD_DOMAIN); + ad.extend_from_slice(&self.metadata.database_id); + ad.extend_from_slice(&self.metadata.schema_version.to_be_bytes()); + ad.push(table_domain); + ad.extend_from_slice(locator); + ad + } +} + +#[derive(Debug)] +struct SqlRow { + rowid: i64, + locator: Vec, + unique_index: Option>, + index_a: Option>, + index_b: Option>, + index_c: Option>, + index_d: Option>, + blob: Vec, +} + +fn sql_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(SqlRow { + rowid: row.get(0)?, + locator: row.get(1)?, + unique_index: row.get(2)?, + index_a: row.get(3)?, + index_b: row.get(4)?, + index_c: row.get(5)?, + index_d: row.get(6)?, + blob: row.get(7)?, + }) +} + +fn insert_record( + conn: &Connection, + domain: u8, + locator: &[u8; 16], + indexes: &[Option>; 5], + sealed: &[u8], +) -> Result { + match conn.execute( + "INSERT INTO store_records + (table_domain, locator, unique_index, index_a, index_b, index_c, index_d, blob) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + domain, + locator.as_slice(), + indexes[0].as_deref(), + indexes[1].as_deref(), + indexes[2].as_deref(), + indexes[3].as_deref(), + indexes[4].as_deref(), + sealed + ], + ) { + Ok(_) => Ok(conn.last_insert_rowid()), + Err(error) if constraint_violation(&error) => Err(StoreError::DuplicateIndex), + Err(error) => Err(StoreError::Db(error)), + } +} + +fn encode_record(logical_key: &[u8], indexes: &IndexKeys, payload: &[u8]) -> Result> { + if logical_key.is_empty() || logical_key.len() > MAX_CANONICAL_KEY_BYTES { + return Err(StoreError::RecordBounds); + } + let key_length = u16::try_from(logical_key.len()).map_err(|_| StoreError::RecordBounds)?; + let mut flags = 0u8; + let mut index_bytes = 0usize; + for (position, key) in indexes.as_array().iter().enumerate() { + if let Some(key) = key { + if key.is_empty() || key.len() > MAX_CANONICAL_KEY_BYTES { + return Err(StoreError::RecordBounds); + } + flags |= 1 << position; + index_bytes = index_bytes + .checked_add(2 + key.len()) + .ok_or(StoreError::RecordBounds)?; + } + } let total = 1usize .checked_add(2) .and_then(|value| value.checked_add(logical_key.len())) + .and_then(|value| value.checked_add(1)) + .and_then(|value| value.checked_add(index_bytes)) .and_then(|value| value.checked_add(payload.len())) .ok_or(StoreError::RecordBounds)?; if total > MAX_RECORD_BYTES { @@ -616,8 +1783,14 @@ fn encode_record(key: &K, payload: &[u8]) -> Result Result> { if version != LOGICAL_RECORD_VERSION { return Err(StoreError::UnsupportedRecordVersion); } - if remainder.len() < 2 { + if remainder.len() < 3 { return Err(StoreError::Serialization); } - let key_len = usize::from(u16::from_be_bytes( + let key_length = usize::from(u16::from_be_bytes( remainder[..2] .try_into() .map_err(|_| StoreError::Serialization)?, )); - let body = &remainder[2..]; - if key_len > body.len() { + let mut body = &remainder[2..]; + if key_length == 0 || key_length > MAX_CANONICAL_KEY_BYTES || body.len() <= key_length { return Err(StoreError::Serialization); } - let (logical_key, payload) = body.split_at(key_len); + let logical_key = &body[..key_length]; + body = &body[key_length..]; + let flags = body[0]; + if flags & !0b1_1111 != 0 { + return Err(StoreError::Serialization); + } + body = &body[1..]; + let mut indexes = [None; 5]; + for (position, slot) in indexes.iter_mut().enumerate() { + if flags & (1 << position) == 0 { + continue; + } + if body.len() < 2 { + return Err(StoreError::Serialization); + } + let length = usize::from(u16::from_be_bytes( + body[..2] + .try_into() + .map_err(|_| StoreError::Serialization)?, + )); + body = &body[2..]; + if length == 0 || length > MAX_CANONICAL_KEY_BYTES || body.len() < length { + return Err(StoreError::Serialization); + } + *slot = Some(&body[..length]); + body = &body[length..]; + } Ok(DecodedRecord { logical_key, - payload, + indexes, + payload: body, }) } -fn validate_logical_key(domain: TableDomain, logical_key: &[u8]) -> Result<()> { - if logical_key.len() != domain.expected_key_len() - || logical_key.first().copied() != Some(domain.key_kind()) - { +fn decoded_indexes_owned(decoded: &DecodedRecord<'_>) -> Result { + let [unique, a, b, c, d] = decoded.indexes; + Ok(IndexKeys { + unique: unique.map(ToOwned::to_owned), + a: a.map(ToOwned::to_owned), + b: b.map(ToOwned::to_owned), + c: c.map(ToOwned::to_owned), + d: d.map(ToOwned::to_owned), + }) +} + +fn validate_index_shape(domain: u8, indexes: &IndexKeys) -> Result<()> { + let shape = indexes.as_array().map(|value| value.is_some()); + let expected = match domain { + MessageRows::DOMAIN => [true, true, false, false, false], + QueueRows::DOMAIN => [false, true, shape[2], shape[3], true], + GroupChainRows::DOMAIN => [false, true, false, false, false], + GroupMessageRows::DOMAIN => [true, true, false, false, false], + MediaObjectRows::DOMAIN => [false, true, false, false, false], + NoteRows::DOMAIN => [true, false, false, false, false], + DeviceSyncRows::DOMAIN => [true, false, false, false, false], + ContactDeviceRows::DOMAIN => [false, true, false, false, false], + MessageDeviceDeliveryRows::DOMAIN => [false, true, true, false, false], + 1..=25 => [false; 5], + MigrationCheckpointRows::DOMAIN => [false; 5], + _ => return Err(StoreError::SchemaMismatch), + }; + if shape != expected { return Err(StoreError::LogicalKeyMismatch); } + for key in indexes.as_array().into_iter().flatten() { + if key.is_empty() || key.len() > MAX_CANONICAL_KEY_BYTES { + return Err(StoreError::RecordBounds); + } + } Ok(()) } -fn validate_locator(domain: TableDomain, locator: &[u8]) -> Result<()> { - let expected = match domain.locator_kind() { - LocatorKind::Equality => 32, - LocatorKind::Random => 16, +fn table_locator_kind(domain: u8) -> Result { + match domain { + IdentityRows::DOMAIN + | SessionRows::DOMAIN + | CapabilityRows::DOMAIN + | SeenRows::DOMAIN + | ReceiptReplayRows::DOMAIN + | ContactRows::DOMAIN + | PrekeyRows::DOMAIN + | ResetRows::DOMAIN + | GroupRows::DOMAIN + | GroupAuthorityRows::DOMAIN + | GroupChainRows::DOMAIN + | MediaTransferRows::DOMAIN + | MediaObjectRows::DOMAIN + | LocalMetadataRows::DOMAIN + | ScheduledRows::DOMAIN + | EphemeralRows::DOMAIN + | DeviceStateRows::DOMAIN + | ContactDeviceRows::DOMAIN + | MessageDeviceDeliveryRows::DOMAIN + | MigrationCheckpointRows::DOMAIN => Ok(LocatorKind::Equality), + MessageRows::DOMAIN + | QueueRows::DOMAIN + | PendingRows::DOMAIN + | GroupMessageRows::DOMAIN + | NoteRows::DOMAIN + | DeviceSyncRows::DOMAIN => Ok(LocatorKind::Random), + _ => Err(StoreError::SchemaMismatch), + } +} + +fn validate_key_for_domain(domain: u8, key: &[u8]) -> Result<()> { + let valid = match domain { + IdentityRows::DOMAIN + | PrekeyRows::DOMAIN + | DeviceStateRows::DOMAIN + | MigrationCheckpointRows::DOMAIN => SingletonKey::validate_encoded(key), + SessionRows::DOMAIN | CapabilityRows::DOMAIN | ContactRows::DOMAIN | ResetRows::DOMAIN => { + AccountKey::validate_encoded(key) + } + MessageRows::DOMAIN => MessageKey::validate_encoded(key), + QueueRows::DOMAIN | PendingRows::DOMAIN => OpaqueRowKey::validate_encoded(key), + SeenRows::DOMAIN | ReceiptReplayRows::DOMAIN | NoteRows::DOMAIN | ScheduledRows::DOMAIN => { + ContentKey::validate_encoded(key) + } + GroupRows::DOMAIN | GroupAuthorityRows::DOMAIN => GroupKey::validate_encoded(key), + GroupChainRows::DOMAIN => GroupMemberKey::validate_encoded(key), + GroupMessageRows::DOMAIN => GroupMessageKey::validate_encoded(key), + MediaTransferRows::DOMAIN | MediaObjectRows::DOMAIN => LocalIdKey::validate_encoded(key), + LocalMetadataRows::DOMAIN => MetadataKey::validate_encoded(key), + EphemeralRows::DOMAIN => EphemeralKey::validate_encoded(key), + DeviceSyncRows::DOMAIN => DigestKey::validate_encoded(key), + ContactDeviceRows::DOMAIN => AccountDeviceKey::validate_encoded(key), + MessageDeviceDeliveryRows::DOMAIN => MessageDeviceKey::validate_encoded(key), + _ => return Err(StoreError::SchemaMismatch), }; - if locator.len() != expected { - return Err(StoreError::SchemaMismatch); + if valid { + Ok(()) + } else { + Err(StoreError::LogicalKeyMismatch) } - Ok(()) +} + +fn domain_uses_opaque_key(domain: u8) -> bool { + matches!(domain, QueueRows::DOMAIN | PendingRows::DOMAIN) } fn seal_metadata( @@ -674,7 +1960,7 @@ fn seal_metadata( rng: &mut impl CryptoRngCore, ) -> Result> { let encoded = postcard::to_allocvec(metadata).map_err(|_| StoreError::Serialization)?; - let mut plain = Vec::with_capacity(1 + encoded.len()); + let mut plain = Zeroizing::new(Vec::with_capacity(1 + encoded.len())); plain.push(METADATA_RECORD_VERSION); plain.extend_from_slice(&encoded); Ok(key.seal(METADATA_AD, &plain, rng)) @@ -691,32 +1977,14 @@ fn open_metadata(key: &StorageKey, sealed: &[u8]) -> Result { if version != METADATA_RECORD_VERSION { return Err(StoreError::Serialization); } - let (metadata, remainder): (DatabaseMetadata, &[u8]) = - postcard::take_from_bytes(encoded).map_err(|_| StoreError::Serialization)?; - if !remainder.is_empty() { - return Err(StoreError::Serialization); - } - Ok(metadata) -} - -fn read_metadata(conn: &Connection, key: &StorageKey) -> Result { - let count: i64 = conn.query_row("SELECT COUNT(*) FROM store_v2_metadata", [], |row| { - row.get(0) - })?; - if count != 1 { - return Err(StoreError::SchemaMismatch); - } - let sealed: Option> = conn - .query_row( - "SELECT blob FROM store_v2_metadata WHERE id = 1", - [], - |row| row.get(0), - ) - .optional()?; - open_metadata(key, &sealed.ok_or(StoreError::SchemaMismatch)?) + decode_exact(encoded) } -fn validate_metadata(metadata: &DatabaseMetadata, physical_version: u32) -> Result<()> { +fn validate_metadata( + metadata: &DatabaseMetadata, + physical_version: u32, + allow_incomplete: bool, +) -> Result<()> { if metadata.schema_version > SCHEMA_VERSION || physical_version > SCHEMA_VERSION { return Err(StoreError::FutureSchema); } @@ -729,27 +1997,42 @@ fn validate_metadata(metadata: &DatabaseMetadata, physical_version: u32) -> Resu if metadata.migrations.is_empty() || metadata.migrations.len() > MAX_MIGRATIONS { return Err(StoreError::InvalidMigrationLedger); } - - let mut seen = BTreeSet::new(); + let mut ids = std::collections::BTreeSet::new(); let mut current = 0u32; - for migration in &metadata.migrations { - if !seen.insert(migration.id) { + for (position, migration) in metadata.migrations.iter().enumerate() { + if !ids.insert(migration.id) { return Err(StoreError::DuplicateMigration); } - if !migration.completed { + if !(migration.completed || allow_incomplete && position + 1 == metadata.migrations.len()) { return Err(StoreError::IncompleteMigration); } - if migration.id != FOUNDATION_MIGRATION_ID { - return Err(StoreError::InvalidMigrationLedger); - } if migration.from_version != current || migration.to_version <= migration.from_version { return Err(StoreError::InvalidMigrationLedger); } current = migration.to_version; } - if current != metadata.schema_version { + let known_fresh = metadata.migrations.len() == 1 + && metadata.migrations[0].id == FRESH_MIGRATION_ID + && metadata.migrations[0].from_version == 0 + && metadata.migrations[0].to_version == SCHEMA_VERSION; + let known_legacy = metadata.migrations.len() == 2 + && metadata.migrations[0].id == LEGACY_MIGRATION_ID + && metadata.migrations[0].from_version == 0 + && metadata.migrations[0].to_version == 1 + && metadata.migrations[1].id == OPAQUE_MIGRATION_ID + && metadata.migrations[1].from_version == 1 + && metadata.migrations[1].to_version == SCHEMA_VERSION; + if (known_fresh && metadata.source_fingerprint.is_some()) + || (known_legacy && metadata.source_fingerprint.is_none()) + { + return Err(StoreError::InvalidMigrationLedger); + } + if (!known_fresh && !known_legacy) || current != metadata.schema_version { return Err(StoreError::InvalidMigrationLedger); } + if !allow_incomplete && metadata.migrations.iter().any(|entry| !entry.completed) { + return Err(StoreError::IncompleteMigration); + } Ok(()) } @@ -768,179 +2051,61 @@ fn validate_preamble(conn: &Connection) -> Result<()> { Ok(()) } -fn physical_schema_version(conn: &Connection) -> Result { - Ok(conn.pragma_query_value(None, "user_version", |row| row.get(0))?) -} - -fn user_object_count(conn: &Connection) -> Result { - Ok(conn.query_row( - "SELECT COUNT(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'", - [], - |row| row.get(0), - )?) -} - -fn validate_table_names(conn: &Connection) -> Result<()> { - let mut statement = conn.prepare( - "SELECT name FROM sqlite_schema - WHERE type = 'table' AND name NOT LIKE 'sqlite_%' - ORDER BY name", - )?; - let tables = statement - .query_map([], |row| row.get::<_, String>(0))? - .collect::>>()?; - if tables != ["store_v2_metadata", "store_v2_records"] { - return Err(StoreError::SchemaMismatch); +fn decode_exact(bytes: &[u8]) -> Result +where + T: for<'de> Deserialize<'de>, +{ + let (value, remainder) = + postcard::take_from_bytes(bytes).map_err(|_| StoreError::Serialization)?; + if !remainder.is_empty() { + return Err(StoreError::Serialization); } - Ok(()) + Ok(value) } -fn validate_duplicate_locators(conn: &Connection) -> Result<()> { - let duplicate: Option = conn - .query_row( - "SELECT 1 FROM store_v2_records - GROUP BY table_domain, locator HAVING COUNT(*) > 1 LIMIT 1", - [], - |row| row.get(0), - ) - .optional()?; - if duplicate.is_some() { - return Err(StoreError::DuplicateIndex); - } - Ok(()) +fn constraint_violation(error: &rusqlite::Error) -> bool { + matches!( + error, + rusqlite::Error::SqliteFailure(inner, _) + if inner.code == ErrorCode::ConstraintViolation + && matches!(inner.extended_code, 1555 | 2067) + ) } -fn validate_columns_and_indexes(conn: &Connection) -> Result<()> { - validate_schema_sql(conn)?; - validate_columns( - conn, - "store_v2_metadata", - &[("id", "INTEGER", 0, 1), ("blob", "BLOB", 1, 0)], - )?; - validate_columns( - conn, - "store_v2_records", - &[ - ("rowid_", "INTEGER", 0, 1), - ("table_domain", "INTEGER", 1, 0), - ("locator", "BLOB", 1, 0), - ("blob", "BLOB", 1, 0), - ], - )?; - - let mut statement = conn.prepare("PRAGMA index_list('store_v2_records')")?; - let indexes = statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(1)?, - row.get::<_, i64>(2)?, - row.get::<_, String>(3)?, - row.get::<_, i64>(4)?, - )) - })? - .collect::>>()?; - if indexes != [("store_v2_record_locator".to_owned(), 1, "c".to_owned(), 0)] { - return Err(StoreError::SchemaMismatch); - } +#[cfg(unix)] +pub(crate) fn protect_file(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; - let mut statement = conn.prepare("PRAGMA index_info('store_v2_record_locator')")?; - let columns = statement - .query_map([], |row| row.get::<_, String>(2))? - .collect::>>()?; - if columns != ["table_domain", "locator"] { - return Err(StoreError::SchemaMismatch); - } + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; Ok(()) } -fn validate_schema_sql(conn: &Connection) -> Result<()> { - let mut statement = conn.prepare( - "SELECT type, name, sql FROM sqlite_schema - WHERE name NOT LIKE 'sqlite_%' ORDER BY name", - )?; - let objects = statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - )) - })? - .collect::>>()?; - let expected = [ - ( - "table".to_owned(), - "store_v2_metadata".to_owned(), - METADATA_TABLE_SCHEMA.to_owned(), - ), - ( - "index".to_owned(), - "store_v2_record_locator".to_owned(), - RECORD_LOCATOR_INDEX_SCHEMA.to_owned(), - ), - ( - "table".to_owned(), - "store_v2_records".to_owned(), - RECORDS_TABLE_SCHEMA.to_owned(), - ), - ]; - if objects != expected { - return Err(StoreError::SchemaMismatch); - } +#[cfg(not(unix))] +pub(crate) fn protect_file(_path: &Path) -> Result<()> { Ok(()) } -fn validate_columns( - conn: &Connection, - table: &str, - expected: &[(&str, &str, i64, i64)], -) -> Result<()> { - let mut statement = conn.prepare(&format!("PRAGMA table_info('{table}')"))?; - let columns = statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, i64>(3)?, - row.get::<_, i64>(5)?, - )) - })? - .collect::>>()?; - let expected = expected - .iter() - .map(|(name, kind, not_null, primary_key)| { - ( - (*name).to_owned(), - (*kind).to_owned(), - *not_null, - *primary_key, - ) - }) - .collect::>(); - if columns != expected { - return Err(StoreError::SchemaMismatch); +pub(crate) fn protect_sqlite_files(path: &Path) -> Result<()> { + protect_file(path)?; + for suffix in ["-wal", "-shm"] { + let mut sidecar = path.as_os_str().to_owned(); + sidecar.push(suffix); + let sidecar = std::path::PathBuf::from(sidecar); + if sidecar.exists() { + protect_file(&sidecar)?; + } } Ok(()) } -fn constraint_violation(error: &rusqlite::Error) -> bool { - matches!( - error, - rusqlite::Error::SqliteFailure(inner, _) - if inner.code == ErrorCode::ConstraintViolation - ) -} - #[cfg(unix)] -fn protect_database_file(path: &Path) -> Result<()> { - use std::os::unix::fs::PermissionsExt; - - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; +pub(crate) fn sync_directory(path: &Path) -> Result<()> { + File::open(path)?.sync_all()?; Ok(()) } #[cfg(not(unix))] -fn protect_database_file(_path: &Path) -> Result<()> { +pub(crate) fn sync_directory(_path: &Path) -> Result<()> { Ok(()) } @@ -950,581 +2115,58 @@ mod tests { use rand::{rngs::StdRng, SeedableRng}; use super::*; - use crate::Store; - - const MASTER_BYTES: [u8; 32] = [0x51; 32]; - - fn master() -> StorageKey { - StorageKey::from_bytes(MASTER_BYTES) - } - - fn target_at(path: &Path, seed: u64) -> (MigrationTarget, StdRng) { - let mut rng = StdRng::seed_from_u64(seed); - let target = MigrationTarget::create( - &MigrationAuthority { private: () }, - path, - &master(), - &mut rng, - ) - .unwrap(); - (target, rng) - } - - fn open_target(path: &Path, master: &StorageKey) -> Result { - MigrationTarget::open(&MigrationAuthority { private: () }, path, master) - } - - fn rewrite_metadata(path: &Path, seed: u64, mutate: impl FnOnce(&mut DatabaseMetadata)) { - let conn = Connection::open(path).unwrap(); - let metadata_key = master().derive(b"KK-store-v2-metadata"); - let sealed: Vec = conn - .query_row( - "SELECT blob FROM store_v2_metadata WHERE id = 1", - [], - |row| row.get(0), - ) - .unwrap(); - let mut metadata = open_metadata(&metadata_key, &sealed).unwrap(); - mutate(&mut metadata); - let mut rng = StdRng::seed_from_u64(seed); - let sealed = seal_metadata(&metadata_key, &metadata, &mut rng).unwrap(); - conn.execute( - "UPDATE store_v2_metadata SET blob = ?1 WHERE id = 1", - params![sealed], - ) - .unwrap(); - } - - fn contains(bytes: &[u8], needle: &[u8]) -> bool { - bytes.windows(needle.len()).any(|window| window == needle) - } - - #[test] - fn authenticated_metadata_indexes_and_random_rows_round_trip() { - let directory = tempfile::tempdir().unwrap(); - let first_path = directory.path().join("first-v2.db"); - let second_path = directory.path().join("second-v2.db"); - let account = AccountKey::new([0x11; 32]); - let message = MessageKey::new([0x11; 32], [0x22; 16]); - - let (first, mut first_rng) = target_at(&first_path, 0x2701); - let (second, mut second_rng) = target_at(&second_path, 0x2702); - assert_eq!(first.schema_version(), SCHEMA_VERSION); - assert_eq!(second.schema_version(), SCHEMA_VERSION); - assert_ne!(first.database_id(), [0; 32]); - assert_ne!(first.database_id(), second.database_id()); - - let contact_index = first.equality_index::(&account); - let capability_index = first.equality_index::(&account); - let other_database_index = second.equality_index::(&account); - assert_ne!(contact_index, capability_index); - assert_ne!(contact_index, other_database_index); - - first - .insert_contact(&account, b"contact payload", &mut first_rng) - .unwrap(); - first - .insert_capability(&account, b"capability payload", &mut first_rng) - .unwrap(); - let first_row = first - .append_message(&message, b"first message", &mut first_rng) - .unwrap(); - let second_row = first - .append_message(&message, b"second message", &mut first_rng) - .unwrap(); - assert_ne!(first_row, second_row); - assert_eq!( - first.contact(&account).unwrap(), - Some(b"contact payload".to_vec()) - ); - assert_eq!( - first.capability(&account).unwrap(), - Some(b"capability payload".to_vec()) - ); - assert_eq!( - first.message(&first_row, &message).unwrap(), - Some(b"first message".to_vec()) - ); - assert_eq!( - first.message(&second_row, &message).unwrap(), - Some(b"second message".to_vec()) - ); - - second - .insert_contact(&account, b"same key", &mut second_rng) - .unwrap(); - drop(first); - drop(second); - - let reopened = open_target(&first_path, &master()).unwrap(); - assert_eq!( - reopened.contact(&account).unwrap(), - Some(b"contact payload".to_vec()) - ); - assert_eq!( - reopened.message(&first_row, &message).unwrap(), - Some(b"first message".to_vec()) - ); - } - - #[test] - fn legacy_entry_points_refuse_the_inactive_target_without_mutation() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("inactive-v2.db"); - let (target, _) = target_at(&path, 0x2703); - drop(target); - let before = std::fs::read(&path).unwrap(); - - assert!(matches!( - Store::open(&path, b"irrelevant"), - Err(StoreError::SchemaMismatch) - )); - let mut rng = StdRng::seed_from_u64(0x2704); - assert!(matches!( - Store::create( - &path, - b"irrelevant", - kult_crypto::KdfProfile { - m_cost_kib: 8, - t_cost: 1, - p_cost: 1, - }, - &mut rng, - ), - Err(StoreError::SchemaMismatch) - )); - assert_eq!(std::fs::read(&path).unwrap(), before); - let reopened = open_target(&path, &master()).unwrap(); - assert_eq!(reopened.schema_version(), SCHEMA_VERSION); - } #[test] - fn future_and_disagreeing_schema_fixtures_fail_closed() { - let directory = tempfile::tempdir().unwrap(); - - let future_physical = directory.path().join("future-physical.db"); - let (target, _) = target_at(&future_physical, 0x2710); - drop(target); - Connection::open(&future_physical) - .unwrap() - .pragma_update(None, "user_version", SCHEMA_VERSION + 1) - .unwrap(); - assert!(matches!( - open_target(&future_physical, &master()), - Err(StoreError::FutureSchema) - )); - - let future_metadata = directory.path().join("future-metadata.db"); - let (target, _) = target_at(&future_metadata, 0x2711); - drop(target); - rewrite_metadata(&future_metadata, 0x2712, |metadata| { - metadata.schema_version = SCHEMA_VERSION + 1; - metadata.migrations[0].to_version = SCHEMA_VERSION + 1; - }); - assert!(matches!( - open_target(&future_metadata, &master()), - Err(StoreError::FutureSchema) - )); - - let older_metadata = directory.path().join("older-metadata.db"); - let (target, _) = target_at(&older_metadata, 0x2713); - drop(target); - rewrite_metadata(&older_metadata, 0x2714, |metadata| { - metadata.schema_version = SCHEMA_VERSION - 1; - metadata.migrations[0].to_version = SCHEMA_VERSION - 1; - }); - assert!(matches!( - open_target(&older_metadata, &master()), - Err(StoreError::SchemaMismatch) - )); - - let older_physical = directory.path().join("older-physical.db"); - let (target, _) = target_at(&older_physical, 0x2715); - drop(target); - Connection::open(&older_physical) - .unwrap() - .pragma_update(None, "user_version", SCHEMA_VERSION - 1) - .unwrap(); - assert!(matches!( - open_target(&older_physical, &master()), - Err(StoreError::SchemaMismatch) - )); - - let drifted_schema = directory.path().join("drifted-schema.db"); - let (target, _) = target_at(&drifted_schema, 0x2716); - drop(target); - Connection::open(&drifted_schema) - .unwrap() - .execute_batch("DROP INDEX store_v2_record_locator") - .unwrap(); - assert!(matches!( - open_target(&drifted_schema, &master()), - Err(StoreError::SchemaMismatch) - )); - - let non_strict_schema = directory.path().join("non-strict-schema.db"); - let (target, _) = target_at(&non_strict_schema, 0x2717); - drop(target); - Connection::open(&non_strict_schema) - .unwrap() - .execute_batch( - "ALTER TABLE store_v2_records RENAME TO store_v2_records_old; - DROP TABLE store_v2_records_old; - CREATE TABLE store_v2_records ( - rowid_ INTEGER PRIMARY KEY, - table_domain INTEGER NOT NULL, - locator BLOB NOT NULL, - blob BLOB NOT NULL - ); - CREATE UNIQUE INDEX store_v2_record_locator - ON store_v2_records (table_domain, locator);", - ) - .unwrap(); + fn logical_record_encoding_is_exact_and_bounded() { + let key = MessageKey::new([1; 32], 0, [2; 16]); + let indexes = IndexKeys::message(&ContentKey::new([2; 16]), &AccountKey::new([1; 32])); + let encoded = encode_record(&key.encode(), &indexes, b"payload").unwrap(); + let decoded = decode_record(&encoded).unwrap(); + assert_eq!(decoded.logical_key, key.encode()); + assert_eq!(decoded.payload, b"payload"); + let content_key = ContentKey::new([2; 16]).encode(); + assert_eq!(decoded.indexes[0], Some(content_key.as_slice())); + let mut future = encoded; + future[0] = 2; assert!(matches!( - open_target(&non_strict_schema, &master()), - Err(StoreError::SchemaMismatch) + decode_record(&future), + Err(StoreError::UnsupportedRecordVersion) )); } #[test] - fn incomplete_duplicate_and_discontinuous_migration_fixtures_fail_closed() { - let directory = tempfile::tempdir().unwrap(); - - let incomplete = directory.path().join("incomplete.db"); - let (target, _) = target_at(&incomplete, 0x2720); - drop(target); - rewrite_metadata(&incomplete, 0x2721, |metadata| { - metadata.migrations[0].completed = false; - }); + fn metadata_accepts_only_fresh_or_legacy_complete_chains() { + let fresh = DatabaseMetadata::fresh([1; 32]); + validate_metadata(&fresh, SCHEMA_VERSION, false).unwrap(); + let mut legacy = DatabaseMetadata::legacy_destination([2; 32], [3; 32], false); assert!(matches!( - open_target(&incomplete, &master()), + validate_metadata(&legacy, SCHEMA_VERSION, false), Err(StoreError::IncompleteMigration) )); - - let duplicate = directory.path().join("duplicate.db"); - let (target, _) = target_at(&duplicate, 0x2722); - drop(target); - rewrite_metadata(&duplicate, 0x2723, |metadata| { - metadata.migrations.push(metadata.migrations[0].clone()); - }); + validate_metadata(&legacy, SCHEMA_VERSION, true).unwrap(); + legacy.migrations[1].completed = true; + validate_metadata(&legacy, SCHEMA_VERSION, false).unwrap(); + legacy.migrations.push(legacy.migrations[1].clone()); assert!(matches!( - open_target(&duplicate, &master()), + validate_metadata(&legacy, SCHEMA_VERSION, false), Err(StoreError::DuplicateMigration) )); - - let discontinuous = directory.path().join("discontinuous.db"); - let (target, _) = target_at(&discontinuous, 0x2724); - drop(target); - rewrite_metadata(&discontinuous, 0x2725, |metadata| { - metadata.migrations[0].from_version = 1; - }); - assert!(matches!( - open_target(&discontinuous, &master()), - Err(StoreError::InvalidMigrationLedger) - )); - - let unknown = directory.path().join("unknown-migration.db"); - let (target, _) = target_at(&unknown, 0x2726); - drop(target); - rewrite_metadata(&unknown, 0x2727, |metadata| { - metadata.migrations[0].id = [0x7f; 16]; - }); - assert!(matches!( - open_target(&unknown, &master()), - Err(StoreError::InvalidMigrationLedger) - )); - - let zero_database = directory.path().join("zero-database.db"); - let (target, _) = target_at(&zero_database, 0x2728); - drop(target); - rewrite_metadata(&zero_database, 0x2729, |metadata| { - metadata.database_id = [0; 32]; - }); - assert!(matches!( - open_target(&zero_database, &master()), - Err(StoreError::SchemaMismatch) - )); - } - - #[test] - fn cross_database_cross_table_and_cross_row_transplants_fail_closed() { - let directory = tempfile::tempdir().unwrap(); - let account = AccountKey::new([0x31; 32]); - - let source_path = directory.path().join("source.db"); - let target_path = directory.path().join("target.db"); - let (source, mut rng) = target_at(&source_path, 0x2730); - source - .insert_contact(&account, b"source", &mut rng) - .unwrap(); - let (target, _) = target_at(&target_path, 0x2731); - drop(source); - drop(target); - let source_conn = Connection::open(&source_path).unwrap(); - let copied: (i64, Vec, Vec) = source_conn - .query_row( - "SELECT table_domain, locator, blob FROM store_v2_records", - [], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .unwrap(); - drop(source_conn); - Connection::open(&target_path) - .unwrap() - .execute( - "INSERT INTO store_v2_records (table_domain, locator, blob) - VALUES (?1, ?2, ?3)", - params![copied.0, copied.1, copied.2], - ) - .unwrap(); - assert!(matches!( - open_target(&target_path, &master()), - Err(StoreError::Crypto(_)) - )); - - let table_path = directory.path().join("cross-table.db"); - let (table_target, mut rng) = target_at(&table_path, 0x2732); - table_target - .insert_contact(&account, b"contact", &mut rng) - .unwrap(); - drop(table_target); - Connection::open(&table_path) - .unwrap() - .execute( - "UPDATE store_v2_records SET table_domain = ?1", - params![TableDomain::Capabilities as u8], - ) - .unwrap(); - assert!(matches!( - open_target(&table_path, &master()), - Err(StoreError::Crypto(_)) - )); - - let row_path = directory.path().join("cross-row.db"); - let (row_target, mut rng) = target_at(&row_path, 0x2733); - row_target - .append_message(&MessageKey::new([1; 32], [1; 16]), b"first", &mut rng) - .unwrap(); - row_target - .append_message(&MessageKey::new([2; 32], [2; 16]), b"second", &mut rng) - .unwrap(); - drop(row_target); - let conn = Connection::open(&row_path).unwrap(); - let rows = conn - .prepare("SELECT rowid_, blob FROM store_v2_records ORDER BY rowid_") - .unwrap() - .query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, Vec>(1)?)) - }) - .unwrap() - .collect::>>() - .unwrap(); - conn.execute( - "UPDATE store_v2_records SET blob = ?1 WHERE rowid_ = ?2", - params![rows[1].1, rows[0].0], - ) - .unwrap(); - conn.execute( - "UPDATE store_v2_records SET blob = ?1 WHERE rowid_ = ?2", - params![rows[0].1, rows[1].0], - ) - .unwrap(); - drop(conn); - assert!(matches!( - open_target(&row_path, &master()), - Err(StoreError::Crypto(_)) - )); - } - - #[test] - fn wrong_master_inner_key_and_record_version_fail_closed() { - let directory = tempfile::tempdir().unwrap(); - - let wrong_master_path = directory.path().join("wrong-master.db"); - let (target, _) = target_at(&wrong_master_path, 0x2740); - drop(target); - assert!(matches!( - open_target(&wrong_master_path, &StorageKey::from_bytes([0x52; 32])), - Err(StoreError::Crypto(_)) - )); - - let wrong_inner_path = directory.path().join("wrong-inner.db"); - let (target, mut rng) = target_at(&wrong_inner_path, 0x2741); - let expected = AccountKey::new([0x41; 32]); - target - .insert_contact(&expected, b"payload", &mut rng) - .unwrap(); - let locator = target.equality_index::(&expected); - let wrong_plain = encode_record(&AccountKey::new([0x42; 32]), b"payload").unwrap(); - let wrong_sealed = target.seal_row(TableDomain::Contacts, &locator, &wrong_plain, &mut rng); - target - .conn - .execute( - "UPDATE store_v2_records SET blob = ?1", - params![wrong_sealed], - ) - .unwrap(); - drop(target); - assert!(matches!( - open_target(&wrong_inner_path, &master()), - Err(StoreError::LogicalKeyMismatch) - )); - - let version_path = directory.path().join("record-version.db"); - let (target, mut rng) = target_at(&version_path, 0x2742); - let key = AccountKey::new([0x43; 32]); - target.insert_contact(&key, b"payload", &mut rng).unwrap(); - let locator = target.equality_index::(&key); - let mut future = encode_record(&key, b"payload").unwrap(); - future[0] = LOGICAL_RECORD_VERSION + 1; - let sealed = target.seal_row(TableDomain::Contacts, &locator, &future, &mut rng); - target - .conn - .execute("UPDATE store_v2_records SET blob = ?1", params![sealed]) - .unwrap(); - drop(target); - assert!(matches!( - open_target(&version_path, &master()), - Err(StoreError::UnsupportedRecordVersion) - )); - } - - #[test] - fn duplicate_indexes_fail_on_typed_write_and_corrupt_open() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("duplicate-index.db"); - let account = AccountKey::new([0x61; 32]); - let (target, mut rng) = target_at(&path, 0x2750); - target.insert_contact(&account, b"first", &mut rng).unwrap(); - assert!(matches!( - target.insert_contact(&account, b"second", &mut rng), - Err(StoreError::DuplicateIndex) - )); - drop(target); - - let conn = Connection::open(&path).unwrap(); - conn.execute_batch("DROP INDEX store_v2_record_locator") - .unwrap(); - let row: (i64, Vec, Vec) = conn - .query_row( - "SELECT table_domain, locator, blob FROM store_v2_records LIMIT 1", - [], - |record| Ok((record.get(0)?, record.get(1)?, record.get(2)?)), - ) - .unwrap(); - conn.execute( - "INSERT INTO store_v2_records (table_domain, locator, blob) - VALUES (?1, ?2, ?3)", - params![row.0, row.1, row.2], - ) - .unwrap(); - drop(conn); - assert!(matches!( - open_target(&path, &master()), - Err(StoreError::DuplicateIndex) - )); } #[test] - fn locked_copy_exposes_only_static_domains_opaque_locators_and_sizes() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("locked-copy.db"); - let account_bytes = *b"locked-copy-contact-key-32-bytes"; - let message_id = *b"row-message-id!!"; - let contact_payload = b"locked-copy-contact-payload"; - let message_payload = b"locked-copy-message-payload"; - let (target, mut rng) = target_at(&path, 0x2760); - let database_id = target.database_id(); - let account = AccountKey::new(account_bytes); - let message_key = MessageKey::new(account_bytes, message_id); - target - .insert_contact(&account, contact_payload, &mut rng) - .unwrap(); - let row_locator = target - .append_message(&message_key, message_payload, &mut rng) - .unwrap(); - let equality_locator = target.equality_index::(&account); - assert_ne!(equality_locator.as_slice(), account_bytes.as_slice()); - assert_ne!(row_locator.as_bytes().as_slice(), message_id.as_slice()); - drop(target); - - let bytes = std::fs::read(&path).unwrap(); - for secret in [ - account_bytes.as_slice(), - message_id.as_slice(), - contact_payload.as_slice(), - message_payload.as_slice(), - database_id.as_slice(), - ] { - assert!(!contains(&bytes, secret)); - } - - let raw = Connection::open(&path).unwrap(); - let columns = raw - .prepare("PRAGMA table_info('store_v2_records')") - .unwrap() - .query_map([], |row| row.get::<_, String>(1)) - .unwrap() - .collect::>>() - .unwrap(); - assert_eq!(columns, ["rowid_", "table_domain", "locator", "blob"]); - let rows = raw - .prepare( - "SELECT table_domain, length(locator), length(blob) - FROM store_v2_records ORDER BY rowid_", - ) - .unwrap() - .query_map([], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, i64>(2)?, - )) - }) - .unwrap() - .collect::>>() - .unwrap(); - assert_eq!(rows.len(), 2); - assert_eq!( - rows.iter().map(|row| (row.0, row.1)).collect::>(), - vec![ - (TableDomain::Contacts as i64, 32), - (TableDomain::Messages as i64, 16), - ] - ); - assert!(rows.iter().all(|row| row.2 > 24 + 16)); - let sealed_metadata: Vec = raw - .query_row("SELECT blob FROM store_v2_metadata", [], |row| row.get(0)) - .unwrap(); - assert!(!contains(&sealed_metadata, &database_id)); + fn random_database_ids_are_nonzero() { + let mut rng = StdRng::seed_from_u64(0x2700); + assert_ne!(random_database_id(&mut rng).unwrap(), [0; 32]); } proptest! { - #![proptest_config(ProptestConfig::with_cases(64))] + #![proptest_config(ProptestConfig::with_cases(512))] #[test] - fn canonical_records_round_trip_without_trailing_ambiguity( - account in any::<[u8; 32]>(), - message_id in any::<[u8; 16]>(), - payload in prop::collection::vec(any::(), 0..4096), + fn arbitrary_logical_record_bytes_fail_closed_without_panicking( + bytes in proptest::collection::vec(any::(), 0..4_096), ) { - let key = MessageKey::new(account, message_id); - let encoded = encode_record(&key, &payload).unwrap(); - let decoded = decode_record(&encoded).unwrap(); - prop_assert_eq!(decoded.logical_key, key.encode()); - prop_assert_eq!(decoded.payload, payload); - } - - #[test] - fn equality_indexes_are_stable_and_table_separated(account in any::<[u8; 32]>()) { - let directory = tempfile::tempdir().unwrap(); - let (target, _) = target_at(&directory.path().join("property.db"), 0x2770); - let key = AccountKey::new(account); - let first = target.equality_index::(&key); - let repeated = target.equality_index::(&key); - let other_table = target.equality_index::(&key); - prop_assert_eq!(first, repeated); - prop_assert_ne!(first, other_table); + let _ = decode_record(&bytes); } } } diff --git a/crates/kult-store/tests/fixtures/schema-v0.1.0.sql b/crates/kult-store/tests/fixtures/schema-v0.1.0.sql new file mode 100644 index 0000000..714419e --- /dev/null +++ b/crates/kult-store/tests/fixtures/schema-v0.1.0.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS identity (id INTEGER PRIMARY KEY CHECK (id = 1), blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS sessions (peer BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS capabilities (peer BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS messages (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS queue (seq INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS seen (id BLOB PRIMARY KEY); +CREATE TABLE IF NOT EXISTS contacts (peer BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS prekeys (id INTEGER PRIMARY KEY CHECK (id = 1), blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS pending (seq INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS resets (peer BLOB PRIMARY KEY); +CREATE TABLE IF NOT EXISTS groups (gid BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS group_authority (gid BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS group_chains (gid BLOB NOT NULL, peer BLOB NOT NULL, blob BLOB NOT NULL, PRIMARY KEY (gid, peer)); +CREATE TABLE IF NOT EXISTS group_msgs (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS media_transfers (id BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS media_objects (id BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS local_metadata (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS note_messages (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS scheduled_messages (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS ephemeral (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS device_state (id INTEGER PRIMARY KEY CHECK (id = 1), blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS device_sync (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS contact_devices (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS message_device_delivery (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); diff --git a/crates/kult-store/tests/fixtures/schema-v0.2.0.sql b/crates/kult-store/tests/fixtures/schema-v0.2.0.sql new file mode 100644 index 0000000..714419e --- /dev/null +++ b/crates/kult-store/tests/fixtures/schema-v0.2.0.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS identity (id INTEGER PRIMARY KEY CHECK (id = 1), blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS sessions (peer BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS capabilities (peer BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS messages (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS queue (seq INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS seen (id BLOB PRIMARY KEY); +CREATE TABLE IF NOT EXISTS contacts (peer BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS prekeys (id INTEGER PRIMARY KEY CHECK (id = 1), blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS pending (seq INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS resets (peer BLOB PRIMARY KEY); +CREATE TABLE IF NOT EXISTS groups (gid BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS group_authority (gid BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS group_chains (gid BLOB NOT NULL, peer BLOB NOT NULL, blob BLOB NOT NULL, PRIMARY KEY (gid, peer)); +CREATE TABLE IF NOT EXISTS group_msgs (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS media_transfers (id BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS media_objects (id BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS local_metadata (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS note_messages (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS scheduled_messages (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS ephemeral (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS device_state (id INTEGER PRIMARY KEY CHECK (id = 1), blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS device_sync (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS contact_devices (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS message_device_delivery (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); diff --git a/crates/kult-store/tests/fixtures/schema-v0.3.0.sql b/crates/kult-store/tests/fixtures/schema-v0.3.0.sql new file mode 100644 index 0000000..863011c --- /dev/null +++ b/crates/kult-store/tests/fixtures/schema-v0.3.0.sql @@ -0,0 +1,26 @@ +CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS identity (id INTEGER PRIMARY KEY CHECK (id = 1), blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS sessions (peer BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS capabilities (peer BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS messages (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS queue (seq INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS seen (id BLOB PRIMARY KEY); +CREATE TABLE IF NOT EXISTS receipt_replay (id BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS contacts (peer BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS prekeys (id INTEGER PRIMARY KEY CHECK (id = 1), blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS pending (seq INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS resets (peer BLOB PRIMARY KEY); +CREATE TABLE IF NOT EXISTS groups (gid BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS group_authority (gid BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS group_chains (gid BLOB NOT NULL, peer BLOB NOT NULL, blob BLOB NOT NULL, PRIMARY KEY (gid, peer)); +CREATE TABLE IF NOT EXISTS group_msgs (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS media_transfers (id BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS media_objects (id BLOB PRIMARY KEY, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS local_metadata (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS note_messages (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS scheduled_messages (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS ephemeral (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS device_state (id INTEGER PRIMARY KEY CHECK (id = 1), blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS device_sync (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS contact_devices (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); +CREATE TABLE IF NOT EXISTS message_device_delivery (rowid_ INTEGER PRIMARY KEY AUTOINCREMENT, blob BLOB NOT NULL); diff --git a/crates/kult-store/tests/local_metadata.rs b/crates/kult-store/tests/local_metadata.rs index 1eed661..0412149 100644 --- a/crates/kult-store/tests/local_metadata.rs +++ b/crates/kult-store/tests/local_metadata.rs @@ -3,7 +3,6 @@ //! copied SQLite database. use rand::{rngs::StdRng, SeedableRng}; -use rusqlite::Connection; use kult_crypto::KdfProfile; use kult_store::{ @@ -127,16 +126,7 @@ fn every_record_round_trips_across_restart_and_stays_sealed() { assert_eq!(store.local_metadata().unwrap(), records); drop(store); - let raw = Connection::open(&db).unwrap(); - let mut statement = raw - .prepare("SELECT blob FROM local_metadata ORDER BY rowid_") - .unwrap(); - let blobs = statement - .query_map([], |row| row.get::<_, Vec>(0)) - .unwrap() - .collect::>>() - .unwrap(); - let joined = blobs.concat(); + let joined = std::fs::read(&db).unwrap(); for secret in [ &b"Expedition plans"[..], &b"Needs reply"[..], @@ -151,9 +141,6 @@ fn every_record_round_trips_across_restart_and_stays_sealed() { "local metadata plaintext leaked into SQLite" ); } - drop(statement); - drop(raw); - let reopened = Store::open(&db, b"pass").unwrap(); assert_eq!(reopened.local_metadata().unwrap(), records); } diff --git a/crates/kult-store/tests/note.rs b/crates/kult-store/tests/note.rs index af366dd..41de397 100644 --- a/crates/kult-store/tests/note.rs +++ b/crates/kult-store/tests/note.rs @@ -40,22 +40,28 @@ fn note_history_survives_restart_without_plaintext_or_peer_columns() { let raw = Connection::open(&database).unwrap(); let columns = raw - .prepare("PRAGMA table_info(note_messages)") + .prepare("PRAGMA table_info(store_records)") .unwrap() .query_map([], |row| row.get::<_, String>(1)) .unwrap() .collect::>>() .unwrap(); - assert_eq!(columns, vec!["rowid_", "blob"]); - let blobs = raw - .prepare("SELECT blob FROM note_messages ORDER BY rowid_") - .unwrap() - .query_map([], |row| row.get::<_, Vec>(0)) - .unwrap() - .collect::>>() + assert_eq!( + columns, + vec![ + "rowid_", + "table_domain", + "locator", + "unique_index", + "index_a", + "index_b", + "index_c", + "index_d", + "blob", + ] + ); + assert!(!std::fs::read(&database) .unwrap() - .concat(); - assert!(!blobs .windows(b"Remember the paper map".len()) .any(|window| window == b"Remember the paper map")); drop(raw); diff --git a/docs/02-threat-model.md b/docs/02-threat-model.md index a0195ef..7da5b0c 100644 --- a/docs/02-threat-model.md +++ b/docs/02-threat-model.md @@ -85,11 +85,14 @@ Malware, forensic seizure of an unlocked device, or a coerced unlock, against a target. **Defense (bounded)**: at-rest encryption under an Argon2id-derived key protects a -powered-off/locked device's sealed record bodies. The current Alpha database -still exposes contact/group identifiers in plaintext lookup columns and does -not bind every ciphertext to its logical row. ADR-0027 has an inactive v2 -destination foundation, but the complete all-table migration remains open P0 -work. Forward secrecy means a captured device does not reveal +powered-off/locked device's sealed record bodies. ADR-0027 replaces sensitive +plaintext SQLite lookup values with database- and domain-separated keyed +indexes, and binds each ciphertext to its final database, schema, table, and +locator identity. A locked copy still exposes approximate row counts/sizes, +insertion order, within-domain equality, access patterns, and change timing. +The implementation has Linux/ext4 test evidence, while independent review and +physical filesystem, power-loss, backup-exclusion, and forensic qualification +remain open. Forward secrecy means a captured device does not reveal messages deleted before capture; post-compromise security means a *transient* compromise is healed by the next DH ratchet step. A persistently compromised endpoint sees everything its user sees; no messenger can prevent that (§5). diff --git a/docs/04-cryptography.md b/docs/04-cryptography.md index 0db5abe..cd2b76c 100644 --- a/docs/04-cryptography.md +++ b/docs/04-cryptography.md @@ -261,13 +261,21 @@ passphrase/biometric-unlocked keystore │ unwraps (XChaCha20-Poly1305) ▼ SK: storage master key - │ HKDF-SHA-256, per-domain info strings + │ HKDF-SHA-256, purpose-separated info strings ▼ - per-table keys: messages / sessions / contacts / queue + metadata / index root / row root / cursor root / media + │ + └─ database + table + index-slot separation ``` -- Database: SQLite; every stored blob individually AEAD-sealed (random 24-B nonce): no - reliance on whole-file encryption alone, and blobs stay sealed in backups. +- Database: SQLite; every stored row is individually AEAD-sealed with a random + 24-byte nonce and bound to its database id, schema, table domain, and final + locator. Guessable equality keys become database/table/slot-separated keyed + indexes rather than plaintext SQLite columns. There is no reliance on + whole-file encryption alone. +- Encrypted backups serialize validated logical records under their independent + mnemonic-derived outer AEAD. They do not copy source database ids, opaque + indexes, SQLite pages, or wrapped database-row ciphertext. - Ratchet session state is the most sensitive record class; serialized state is additionally wrapped and zeroized in memory after each persist. - Key rotation: re-wrap `SK` under a new KEK (passphrase change is O(1)); full `SK` diff --git a/docs/07-storage.md b/docs/07-storage.md index 715293d..be70fa4 100644 --- a/docs/07-storage.md +++ b/docs/07-storage.md @@ -13,12 +13,13 @@ locally deletable within explicit endpoint and copy-retention limits. an operator. Implemented C2 sync is explicit device-to-device, end-to-end encrypted, and accepted only between account-authorized physical devices. -2. **Durable record bodies at rest are sealed.** The core database does not - persist message, draft, or media plaintext. The current Alpha schema still - has plaintext equality and ordering columns that expose some exact - relationship metadata in a copied database; these are documented below and - are not yet replaced. ADR-0027 now has an inactive, independently testable - v2 destination foundation, but no current user table enters that format. +2. **Durable records and sensitive equality keys at rest are sealed.** The core + database does not persist message, draft, or media plaintext. Contact, + group, delivery-token, message, media, device, and other guessable equality + identifiers appear only inside row ciphertext or as database- and + domain-separated keyed indexes. SQLite still reveals static schema + structure, approximate row counts and sizes, insertion order, and access + patterns. Bounded protected transients required by OS picker, recording, editing, playback, or explicit export workflows are temporary exceptions with lifecycle cleanup, never durable sources of truth. @@ -43,8 +44,8 @@ HKDF per-domain keys. | Domain | Contents | Notes | |---|---|---| | `identity` | Own keys (wrapped), device settings | Smallest, most sensitive; extra wrap layer | -| `sessions` | Serialized ratchet states, skipped-key store | Body sealed and rewrapped on persist; current peer-key lookup column remains plaintext metadata | -| `contacts` | Peer keys, verification state, petnames, relay hints | Body sealed; current peer-key lookup column exposes an exact identifier in a copied database; selected fields may enter authenticated own-device sync | +| `sessions` | Serialized ratchet states, skipped-key store | Body sealed and rewrapped on persist; peer lookup is a keyed opaque locator | +| `contacts` | Peer keys, verification state, petnames, relay hints | Body sealed; peer lookup is a keyed opaque locator; selected fields may enter authenticated own-device sync | | `devices` | Own and contact device certificates, signed manifests, revocation tombstones | Bounded authority state; exact physical identities | | `device_sync` | Per-device channels, counters, Lamport winners, terminal convergence tombstones | Direction-bound, replay-protected, never a cloud log | | `messages` | Envelope plaintexts post-decrypt, delivery state | Per-blob AEAD, random nonces | @@ -56,21 +57,36 @@ HKDF per-domain keys. | `ephemeral` | Exact local deadlines, mode, transfer references, active/terminal lifecycle | Sealed separately; terminal tombstones block resurrection after plaintext/media deletion | | `local_metadata` | Conversation types, folders, pins, labels, drafts, UI preferences, custom icons | Endpoint-private; only the C2 allowlist can sync to another owned device | -Every record body is individually AEAD-sealed (XChaCha20-Poly1305, random -24-byte nonce, table name + row purpose as associated data). The current Alpha -schema is **not** an opaque-index store: contact/device public keys, group ids, -and related equality columns remain plaintext, so a copied locked database can -reveal exact relationship identifiers in addition to row counts and sizes. -Constant per-table associated data prevents cross-table/cross-key opening but -does not bind every ciphertext to its logical row, so a writer can substitute -some valid rows within one table without guaranteed detection. -[ADR-0027](adr/0027-opaque-indexed-store.md) defines the keyed indexes, -row-bound associated data, versioned migration, and deletion wording required -before stronger at-rest metadata claims. Its implemented foundation exercises -the destination metadata, index, record, and row-binding formats behind an -inactive migration boundary. The complete all-table migration and replacement -protocol remains open, so these protections do not describe current user -databases. +Every database has a random identity. Every record body is individually +AEAD-sealed with XChaCha20-Poly1305 and a random 24-byte nonce. Associated data +binds the row to that database identity, physical schema version, logical table +domain, and final locator. The decoded envelope repeats its logical key and +secondary keys; opening recomputes and verifies the keyed locator and every +opaque SQLite index before returning the payload. + +The physical schema is generic: metadata plus one record table and fixed opaque +index slots serve all 25 sensitive logical domains. Pairwise and group history +use keyed message-id and conversation indexes, bounded authenticated cursor +pages, and indexed exact edit/delete. A copied locked database can still reveal +row counts, approximate sizes, insertion order, equality and access patterns +within one index domain, and change timing; it does not provide plaintext +relationship identifiers as SQLite keys. See +[ADR-0027](adr/0027-opaque-indexed-store.md) and the +[opaque-store qualification](33-opaque-store-qualification.md). + +Opening a released plaintext-index schema validates it against the retained +release fixture, decrypts and verifies bounded rows, and copies logical records +in 256-row transactions into a fresh-id sibling database. A sealed checkpoint +supports restart. Counts, references, SQLite integrity, every opaque row, and a +complete reopen are checked before the synced sibling atomically replaces the +active path. A synced rollback copy remains until that reopen succeeds. + +Deletion removes the live logical row. An explicit bounded maintenance call can +enable SQLite full secure-delete, incrementally vacuum at most 4,096 pages, and +truncate an observed WAL no larger than the caller's bound. The operation +reports deferral and always reports that forensic erasure is not guaranteed. +These controls reduce live-file remnants but cannot control prior copies, +snapshots, flash remapping, recipient devices, or operating-system artifacts. B9 formatting creates no additional durable state. The `messages`, `scheduled_messages`, group history, and note-to-self rows retain exact source @@ -206,7 +222,8 @@ designed separately. ## 3. Search (planned) -The current Alpha does not ship a durable full-text index. ADR-0027 proposes a +The current Alpha does not ship a durable full-text index. A future design may +use a separately keyed, bounded local index in which tokenized terms are HMAC'd under a search-domain key before insertion. That design avoids plaintext vocabulary at rest but still leaks repeated-term equality and access patterns inside one @@ -229,7 +246,12 @@ before search is described as available. revokes every device active in the backup, and mints a fresh sole active physical device; sessions re-handshake (ratchet states and reusable device private credentials are deliberately *not* portable). Format - and mechanism: ADR-0011. + and mechanism: ADR-0011. After outer backup decryption, the payload is a + bounded stream of logical records; it contains no source database id, opaque + index, SQLite page, or wrapped database-row ciphertext. Restore validates the + complete logical payload and references, writes a fresh-id sibling database, + syncs and atomically installs it, and then fully reopens it. It never exposes + a partially restored destination as a valid store. - **B18 label backup behavior**: `KKR7` preserves exact label IDs, names, color tokens, insertion order, assignments, and stale-reference behavior. Labels have no cloud, server, contact, or taxonomy-sharing path; C2 may converge them @@ -292,9 +314,9 @@ before search is described as available. bodies in the core database or encrypted Komms backups, or intentionally in logs, analytics, crash metadata, or notification metadata. Explicit warned plaintext export and protected lifecycle-bound application transients are the - narrow exceptions described above. The current plaintext relationship indexes - remain the separately disclosed Alpha limitation; logs remain structured and - content-free by policy. + narrow exceptions described above. Keyed equality values and row ciphertext + remain inside Komms-owned storage; logs remain structured and content-free by + policy. - Message keys after use; chain keys after advancing (zeroize-on-drop). - Plaintext contact graphs intentionally uploaded to a remote system. Optional relay queues hold only sealed envelopes under rotating tokens with TTLs, but diff --git a/docs/24-local-release-gate.md b/docs/24-local-release-gate.md index dac5b24..5352bdd 100644 --- a/docs/24-local-release-gate.md +++ b/docs/24-local-release-gate.md @@ -38,14 +38,16 @@ The script runs: 1. workspace formatting, all-target/all-feature warnings-as-errors clippy, all tests, `no_std` crypto/protocol builds, and `cargo-deny`; -2. the desktop workspace's independent format, clippy, test, and deny gates; -3. generated Kotlin UniFFI bindings plus the Android JVM/core two-node suite; -4. generated Swift UniFFI bindings plus the iOS/macOS host two-node suite; -5. Android APK/lint and the unsigned iOS Simulator application build when their +2. the ADR-0027 100,000- and 1,000,000-message migration, unlock, indexed page, + exact edit/delete, memory, and database-growth budgets; +3. the desktop workspace's independent format, clippy, test, and deny gates; +4. generated Kotlin UniFFI bindings plus the Android JVM/core two-node suite; +5. generated Swift UniFFI bindings plus the iOS/macOS host two-node suite; +6. Android APK/lint and the unsigned iOS Simulator application build when their complete SDKs are installed; -6. every crypto and protocol fuzz target for 60 seconds, including C2 device +7. every crypto and protocol fuzz target for 60 seconds, including C2 device records and C7 call-control/call-media parsers; and -7. final Git whitespace and worktree review. +8. final Git whitespace and worktree review. Run from the repository root: @@ -84,12 +86,12 @@ Hosted automation complements the local checkpoint: - `.github/workflows/ci.yml` repeats core/desktop format, lint, tests, `no_std`, dependency policy, fuzz smoke, generated Android/iOS host suites, - MSRV 1.88, and Android debug-APK assembly; + MSRV 1.88, Windows core-storage tests, and Android debug-APK assembly; - the iOS Simulator job remains gated by the `IOS_APP_CI=1` repository variable; it is enabled for the current per-push release evidence; - `.github/workflows/audit.yml` runs weekly and on demand: advisories for both - Cargo workspaces, core tests on macOS, and an informational coverage snapshot; - and + Cargo workspaces, core tests on macOS, the opaque-store scale gate on Linux, + and an informational coverage snapshot; and - `.github/workflows/hil-nightly.yml` remains dormant until a trusted `meshtastic-hil` bench is online and `HIL_BENCH=armed`. diff --git a/docs/29-stabilization-program.md b/docs/29-stabilization-program.md index a0fa8ea..a20506a 100644 --- a/docs/29-stabilization-program.md +++ b/docs/29-stabilization-program.md @@ -239,7 +239,7 @@ This table prevents a prior concern from disappearing into roadmap prose. | Unix store writer exclusion now combines a no-follow sidecar with a database-inode lock; equivalent alias resistance and hostile-filesystem qualification remain open on other supported platforms | P0-06, P0-09 | | The Unix RPC sidecar is no-follow and owner-only, but portable stale-socket replacement still requires a daemon-owned parent directory to exclude hostile rename/unlink races | P0-07, P0-09 | | Current linked devices copy the account root, so a compromised revoked device can mint a replacement credential; ADR-0026 requires offline-root migration and majority-authorized manifests | P0-03, P0-06 | -| Current SQLite equality columns reveal exact contact/group identifiers in a locked copy and constant row AD permits same-table substitution; ADR-0027 has an inactive v2 destination foundation, while the complete all-table migration remains open | P0-03, P0-06 | +| ADR-0027 removes sensitive plaintext SQLite equality identifiers and binds every sealed row, but independent storage review and real macOS, Windows, mobile, power-loss, backup-exclusion, and forensic qualification remain open | P0-03, P0-06, P0-09 | | Current sender-key group content proves membership, not individual origin, so a malicious member can forge another member's text, edit, vote, or ephemeral event; ADR-0029 requires recipient-verifiable group origins | P0-03, P0-06 | | RAM-only storage, disabled logs, and aggregate metrics reduce retention but remain deployment controls; a cloud operator can still observe network metadata, running memory, and availability | P0-01, P0-04, P1-07 | diff --git a/docs/31-release-evidence-ledger.md b/docs/31-release-evidence-ledger.md index bef5ee9..b019a2c 100644 --- a/docs/31-release-evidence-ledger.md +++ b/docs/31-release-evidence-ledger.md @@ -41,7 +41,7 @@ accepts it and the assignment is recorded in | **P0-06 Independent crypto and protocol assurance** | Andri (interim SEC); independent cryptography reviewer: **Unassigned**; independent interoperability implementer: **Unassigned** | Automated evidence for local KATs, properties, sessions, and protocol decoding. | **Open.** No external vectors, separate implementation, review scope, findings report, disposition, or residual-risk statement. P0 protocol/security ADRs remain proposed. | [Baseline CI](https://github.com/AndriGitDev/Komms/actions/runs/30199264838); [Crypto KATs](../crates/kult-crypto/tests/kat.rs); [properties](../crates/kult-crypto/tests/properties.rs); [session tests](../crates/kult-crypto/tests/session.rs); [cryptography spec](04-cryptography.md) | 2026-08-09 | | **P0-07 Signed and recoverable distribution** | Andri (interim REL/SEC); independent release evaluator: **Unassigned** | Implemented Alpha packaging with checksums; some workflow provenance/SBOM paths exist. | **Open.** Stable desktop/mobile signing, protected release-key recovery, authenticated updates, reproducibility measurements, clean install/upgrade/rollback, store/repository publication, and external verification are missing. | [Release runbook](25-release-runbook.md); [release workflow](../.github/workflows/release.yml); [0.3 Alpha artifacts](https://github.com/AndriGitDev/Komms/releases/tag/v0.3.0) | 2026-08-09 | | **P0-08 Durable mailbox and operator qualification** | Andri (interim NET/SEC); independent operator evaluator: **Unassigned** | Implemented Alpha mailbox-v1 behavior with automated bounds and end-to-end receipt tests. | **Open.** Mailbox v1 can delete before endpoint custody. Persistent v2 deposits, leases, restart/disk-full/overload/expiry/multi-operator evidence, published operator policy, and maintained self-hosting cost model are missing. | [ADR-0032](adr/0032-leased-mailbox-delivery.md); [node mailbox tests](../crates/kult-node/tests/mailbox_e2e.rs); [transport mailbox tests](../crates/kult-transport/tests/mailbox.rs) | 2026-08-09 | -| **P0-09 Field qualification across supported claims** | Andri (interim PROD/NET/REL); independent accessibility/field evaluator: **Unassigned** | Implemented shells and automated host, simulator, and packaging evidence. | **Open.** No published named-device/OS/NAT/background/handoff/accessibility/recovery/two-radio matrix. macOS and Linux stable support cells are not frozen. | [Local release gate](24-local-release-gate.md); [HIL bench](10-hil-bench.md); [candidate platform rule](30-stable-v1-product-profile.md#1-installation-and-supported-systems) | 2026-08-09 | +| **P0-09 Field qualification across supported claims** | Andri (interim PROD/NET/REL); independent accessibility/field evaluator: **Unassigned** | Implemented shells with host, simulator, packaging, and hosted Windows/NTFS core-storage evidence. | **Open.** No published named-device/OS/NAT/background/handoff/accessibility/recovery/two-radio matrix. macOS and Linux stable support cells are not frozen. | [Local release gate](24-local-release-gate.md); [opaque-store qualification](33-opaque-store-qualification.md); [HIL bench](10-hil-bench.md); [candidate platform rule](30-stable-v1-product-profile.md#1-installation-and-supported-systems) | 2026-08-09 | | **P0-10 Accountable founder authority, review, and incidents** | Andri (FND; interim COM/SEC); independent reviewers and backup steward: **Unassigned** | Implemented as public governance, ownership, security intake, recusal, release authority, and incident policy. | **Open.** No accepted backup steward, independent sensitive-surface reviewers, rehearsed incident record, or continuity handoff. Founder self-review is not independent. | [Governance](../GOVERNANCE.md); [maintainers](../MAINTAINERS.md); [security and incidents](../SECURITY.md); [CODEOWNERS](../.github/CODEOWNERS); [ADR-0033](adr/0033-nonprofit-founder-stewardship.md) | 2026-08-09 | ## 2. Stable public claim register @@ -66,7 +66,7 @@ the listed gap. A new stable claim requires a new identifier here. | **SV1-C11 Supported platforms** | The stable release supports only the named device, OS, architecture, lifecycle, install, upgrade, and accessibility cells published as passed. | Andri (PROD/REL); independent field/accessibility evaluator: **Unassigned** | Implemented shells with automated build/simulator evidence | [Platform rule](30-stable-v1-product-profile.md#1-installation-and-supported-systems); [release gate](24-local-release-gate.md) | P0-09 named physical matrix and exact macOS/Linux support cells | 2026-08-09 | | **SV1-C12 Replaceable services** | Standard defaults are disclosed and replaceable; no project service is the user identity authority or receives message plaintext or user identity private keys. | Andri (NET/SEC/FND); external operator reviewer: **Unassigned** | Designed; pure-core components implemented | [Stabilization contract](29-stabilization-program.md#1-product-and-architecture-contract); [ADR-0034](adr/0034-operator-minimized-reference-discovery.md); [self-hosting](26-self-hosting.md) | Dedicated reference services, public configuration/revision, default-blackhole and replacement evidence, independent operator review | 2026-08-09 | | **SV1-C13 Resilient retry and fallback** | Komms retains queued work, retries within declared bounds, and can use more than one supported route; it does not guarantee availability or delivery time. | Andri (NET/PROD); external field evaluator: **Unassigned** | Automated evidence | [Transport design](05-transports.md); [node delivery tests](../crates/kult-node/tests/node_e2e.rs); [mesh policy tests](../crates/kult-node/tests/mesh_policies.rs); [sneakernet tests](../crates/kult-transport/tests/sneakernet.rs) | Real NAT, background, handoff, operator-failure, and two-radio field matrix | 2026-08-09 | -| **SV1-C14 Local deletion limits** | Delete removes live logical history and Komms-owned references from the current profile; it does not promise forensic or remote erasure. | Andri (SEC/PROD); external storage reviewer: **Unassigned** | Implemented with automated evidence in current schema; inactive v2 destination foundation implemented | [Storage limits](07-storage.md); [ADR-0027](adr/0027-opaque-indexed-store.md); [v2 destination foundation](../crates/kult-store/src/store_v2.rs); [ephemeral tests](../crates/kult-node/tests/ephemeral_e2e.rs) | Complete all-table conversion and atomic replacement, migration/remnant qualification, supported-platform storage review | 2026-08-09 | +| **SV1-C14 Local deletion limits** | Delete removes live logical history and Komms-owned references from the current profile; it does not promise forensic or remote erasure. | Andri (SEC/PROD); external storage reviewer: **Unassigned** | Implemented with repeatable schema, migration, restore, remnant-control, Linux/ext4 scale, and hosted Windows Server 2025/NTFS storage-suite evidence | [Storage limits](07-storage.md); [ADR-0027](adr/0027-opaque-indexed-store.md); [qualification](33-opaque-store-qualification.md); [Windows/NTFS CI](https://github.com/AndriGitDev/Komms/actions/runs/30225556928); [opaque store](../crates/kult-store/src/store_v2.rs); [migration](../crates/kult-store/src/migration.rs); [backup tests](../crates/kult-store/tests/backup.rs); [ephemeral tests](../crates/kult-node/tests/ephemeral_e2e.rs) | Independent storage review; physical macOS, Windows, Android, and iOS qualification; sudden-power-loss, backup-exclusion, permissions, snapshot, and forensic evidence | 2026-08-09 | | **SV1-C15 Nonprofit project mission** | Official Komms activity follows a nonprofit public-benefit mission; this is project policy, not registered-charity status and not a restriction on independent AGPL commercial use. | Andri (FND/COM; project policy owner); qualified legal reviewer: **Unassigned** | Implemented governance policy | [ADR-0033](adr/0033-nonprofit-founder-stewardship.md); [governance](../GOVERNANCE.md) | Legal entity, funding/trademark/asset policies, and qualified policy review remain P1 work | 2026-10-26 | ## 3. Evidence-level rules diff --git a/docs/33-opaque-store-qualification.md b/docs/33-opaque-store-qualification.md new file mode 100644 index 0000000..39dc759 --- /dev/null +++ b/docs/33-opaque-store-qualification.md @@ -0,0 +1,187 @@ +# 33: Opaque Store Qualification + +This record separates implemented storage guarantees from filesystem and +forensic claims that still need real platform evidence. It covers the +ADR-0027 schema, legacy migration, encrypted backup restore, local remnant +maintenance, and large-history budgets. + +## 1. Locked-database contract + +The active schema has three application tables: `store_bootstrap`, +`store_metadata`, and `store_records`. The six application indexes cover a +domain plus locator, unique lookup, or one of four secondary index slots. +Sensitive logical domains do not have their own SQLite columns or indexes. + +The record table serves these 25 domains: + +- identity, sessions, capabilities, messages, queue, seen ids, receipt replay, + contacts, prekeys, pending envelopes, and session resets; +- groups, group authority, group chains, and group messages; +- media transfers, media objects, local metadata, note messages, scheduled + messages, and ephemeral state; and +- device state, device sync, contact devices, and per-device message delivery. + +An equality locator is HMAC-SHA-256 under a key separated by database, table, +and locator purpose. Append-style rows use random 16-byte locators. Secondary +indexes use separately derived purposes, so a value cannot be correlated +between databases, domains, or index slots. + +Every row's XChaCha20-Poly1305 associated data includes the final database id, +physical schema version, logical table domain, and final locator. The sealed +logical envelope repeats its record version, logical key, secondary keys, and +payload. Opening a store checks that the decoded key reproduces its locator, +that decoded secondary keys reproduce every SQLite index, and that the decoded +payload agrees with the key for that record type. + +Pairwise and group history use keyed unique message-id indexes and keyed +conversation indexes. Pages are limited to 512 records and use a 57-byte +authenticated cursor bound to the database, domain, conversation, and last +row. Exact edit and delete first use the unique index, then verify the decoded +logical identity before changing the located row. + +Retained tests cover locked-copy identifier inspection, valid-ciphertext +logical-key and secondary-index mismatch, database/table/locator transplant, +cursor forgery and cross-context reuse, stale cursors, and SQLite query plans +for page and exact-id lookups. Arbitrary bounded logical-envelope bytes are +also decoded under a no-panic property test. + +## 2. Released-schema migration + +Released SQL fixtures are retained for `v0.1.0`, `v0.2.0`, and `v0.3.0`. +Migration accepts only an exact released layout. Each table has a strict row +count and value-type contract; individual ciphertexts, total rows, and decoded +records are bounded before allocation or insertion. Plaintext legacy keys must +equal the identities inside their decoded records. + +The source is validated and fingerprinted before replacement. A new private +sibling database receives a fresh random database id. Copying uses 256-row +transactions and a sealed checkpoint containing the source fingerprint, next +table, last row id, and copied counts. A restart resumes only if the current +source and checkpoint still agree. + +Group authority and chains, media object ownership, contact devices, and +per-device delivery rows receive referential validation. Completion verifies +per-table counts, SQLite integrity, absence of the checkpoint, the authenticated +migration ledger, every opaque row, and a complete reopen. + +Before work starts, free space must cover three times the source database and +WAL size, 1,024 bytes per source row, and a 64 MiB reserve. The completed +sibling is checkpointed out of WAL mode, synced, and directory-synced. The +source is separately checkpointed and copied to a synced rollback sibling. +Replacement then uses the platform's atomic replace operation. The containing +directory is synced before the new database is reopened and fully validated; +only then is the rollback copy removed and that removal directory-synced. + +Five replacement interruption points cover: + +1. completed sibling before rollback creation; +2. synced rollback before replacement; +3. replacement before directory sync; +4. directory sync before reopen; and +5. validated reopen before rollback cleanup. + +Restart tests prove that each phase either resumes migration, opens the complete +new store, or restores the valid rollback copy. Malformed type, oversized row, +logical-key mismatch, missing reference, changed source, incomplete checkpoint, +count mismatch, and replacement-recovery tests fail closed. Early validation +failure leaves the active legacy path untouched and does not create a rollback +copy. + +These tests exercise process interruption at each named boundary. They are not +evidence of behavior under sudden power loss, controller write-cache failure, +filesystem corruption, snapshot rollback, or flash remapping. + +## 3. Backup and restore boundary + +Encrypted `KKR1` through `KKR7` files decode into versioned logical records. +The decrypted backup payload contains the intended portable identity, contact, +history, group, metadata, note, tombstone, and linked-device records. It never +contains the source database id, keyed locators or indexes, SQLite pages, or +wrapped opaque-store row ciphertext. + +Restore validates the complete logical payload, record counts, duplicates, +bounds, and references before creating the destination. Its free-space check +requires twice the decoded logical bytes, 2,048 bytes per restored record, and +a 64 MiB reserve. It writes a fresh private sibling with a fresh database id, +performs full store validation, checkpoints and syncs the file, directory-syncs, +atomically installs it only at an absent destination, directory-syncs again, +and reopens it. + +Three restore interruption points cover the synced sibling, installed file +before directory sync, and directory-synced file before reopen. Restart tests +prove that a partial sibling is either rejected and honestly cleaned up or a +fully installed store opens normally. Restore never silently clobbers an +existing destination. + +## 4. Bounded deletion maintenance + +Logical deletion removes the live row and application references. The +maintenance API enables full SQLite `secure_delete`, requests at most 4,096 +incremental-vacuum pages per call, and accepts a WAL checkpoint bound no larger +than 256 MiB. It truncates the WAL only when the observed file fits the caller's +bound and no reader prevents completion. The report includes freelist and WAL +measurements, whether truncation was deferred, and a value that is always false +for forensic-erasure guarantee. + +SQLite cleanup can reduce remnants in the live Komms files. It cannot erase a +prior copy, filesystem snapshot, recipient copy, SSD-remapped block, carrier +ciphertext, or operating-system artifact. User copy therefore describes +logical removal from this Komms history, not physical or remote erasure. + +## 5. Large-history budgets and results + +`scripts/store-scale-gate.sh` creates the released `v0.3.0` schema, inserts +sealed legacy messages, migrates it, fully reopens and validates it, reads a +64-row conversation page, edits one middle message by exact id, deletes one +message by exact id, and measures SQLite main/WAL growth. It runs both required +sizes with the mobile Argon2id profile. + +The budgets are: + +| Rows | Migration | Unlock | Page | Exact edit | Exact delete | Peak RSS increase | Database + WAL | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 100,000 | 180 s | 30 s | 250 ms | 250 ms | 250 ms | 512 MiB | 64 MiB + 1,024 B/row | +| 1,000,000 | 1,800 s | 180 s | 500 ms | 500 ms | 500 ms | 768 MiB | 64 MiB + 1,024 B/row | + +The 2026-07-26 Linux qualification measured commit `23c423c` on x86_64, +kernel 6.12.13, ext4, and Rust 1.88.0. Both sizes passed: + +| Rows | Migration | Unlock | Page | Exact edit | Exact delete | Peak RSS increase | Database + WAL | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 100,000 | 10.824 s | 1.171 s | 378 µs | 3.021 ms | 1.232 ms | 70.62 MiB | 49.92 MiB | +| 1,000,000 | 143.641 s | 10.425 s | 388 µs | 2.575 ms | 1.120 ms | 70.50 MiB | 501.14 MiB | + +Timing results describe this host and source revision, not every supported +device. The enforced budgets are regression ceilings rather than product +latency promises. + +## 6. Platform qualification + +| Platform/filesystem | Evidence in this change | Still open | +|---|---|---| +| Linux/ext4 | Owner-only database, WAL, SHM, lock, and media-directory mode tests; index/privacy tests; all migration and restore interruption points; real file and directory sync calls; atomic same-directory replacement; both scale sizes | Sudden-power-loss rig, controller/cache faults, snapshots, backup-exclusion integration, and forensic examination | +| macOS/APFS | Shared Unix permission, sync, and replacement implementation; scheduled macOS core test is configured | A result for this revision, power-loss behavior, app-container/backup exclusion, snapshots, and forensic examination | +| Windows Server 2025/NTFS (hosted) | [CI run 30225556928](https://github.com/AndriGitDev/Komms/actions/runs/30225556928) on commit `d8b328e` explicitly identified the checkout volume as NTFS, then passed the complete storage suite: released-schema migration, logical backup/restore, every replacement interruption point, opaque-index and privacy checks, exact history operations, and cross-process writer exclusion. The replacement path uses replace-existing and write-through semantics. | Owner-only ACL enforcement, directory durability under sudden power loss, physical storage behavior, backup integration, snapshots, and forensic examination | +| Android/iOS app filesystems | The Rust storage path is shared with the Unix implementation | Physical-device migration/restore, lifecycle interruption, app-private permissions, cloud-backup exclusion, free-space behavior, power-loss behavior, and forensic examination | + +No unsupported cell is promoted by inference. The hosted Windows result +establishes code-path compatibility for that exact revision and identified +filesystem. It does not close physical filesystem, backup, power-loss, +permissions, snapshot, or forensic cells. A hosted macOS result remains open. + +## 7. Repeatable commands + +The focused storage checks are: + +```sh +cargo test -p kult-store --lib -- --test-threads=1 +cargo test -p kult-store --test backup -- --test-threads=1 +scripts/store-scale-gate.sh +``` + +The complete publication candidate runs +`scripts/local-release-matrix.sh`, which includes the scale gate, workspace and +desktop Rust checks, backup/restore tests, parser fuzz targets, platform host +checks where their SDKs exist, dependency policy, and final repository review. + +The remaining open evidence above is intentionally not represented as passed. diff --git a/docs/adr/0027-opaque-indexed-store.md b/docs/adr/0027-opaque-indexed-store.md index 83563ab..4e3db89 100644 --- a/docs/adr/0027-opaque-indexed-store.md +++ b/docs/adr/0027-opaque-indexed-store.md @@ -1,21 +1,21 @@ # ADR-0027: Versioned opaque indexes and row-bound sealed storage -- **Status**: Proposed; inactive v2 foundation implemented +- **Status**: Accepted; implemented - **Date**: 2026-07-26 ## Context -Komms seals record bodies, but the current SQLite schema uses plaintext account -public keys and group ids as primary keys. A copied locked database therefore -reveals exact contact and group relationships. Several tables also seal every -row under constant associated data, so a local database writer can transplant -valid ciphertext between identifiers without the store proving that the inner -record belongs to the requested row. +The released pre-v2 SQLite schemas used plaintext account public keys, group +ids, delivery tokens, message ids, media ids, and other guessable equality +identifiers. A copied locked database therefore revealed exact relationships. +Several tables also sealed every row under constant associated data, so a local +database writer could transplant valid ciphertext between identifiers without +the store proving that the inner record belonged to the requested row. -The schema has no versioned migration ledger. Additive `CREATE TABLE IF NOT -EXISTS` calls cannot safely express index replacement, authenticated row -identity, or large-history query plans. Updating or deleting a message scans -and decrypts the full history. +Those schemas had no versioned migration ledger. Additive `CREATE TABLE IF NOT +EXISTS` calls could not safely express index replacement, authenticated row +identity, or large-history query plans. Updating or deleting a message scanned +and decrypted the full history. SQLite row deletion and flash storage do not support an honest guarantee that deleted bytes are physically unrecoverable. The design must protect a locked @@ -172,8 +172,9 @@ offer stronger local guarantees, but it must be measured and narrowly stated. social-graph indexes. - Every store operation and backup/restore path must handle explicit record and schema versions. -- Migration requires temporary space approximately equal to another database - and must surface that requirement before starting. +- Migration requires space for the source, sibling destination, rollback copy, + row overhead, and a fixed reserve; it must surface that requirement before + starting. - Database replacement and old-file cleanup need platform-specific qualification. - Marketing loses an absolute deletion slogan and gains a claim the @@ -181,23 +182,40 @@ implementation can defend. ## Implementation status -The first independently safe slice is implemented as an inactive v2 migration -destination in `kult-store`. It provides authenticated metadata with a fresh -database id, an explicit physical and authenticated schema version, and a -completed migration ledger. Its typed contact, capability, and message fixture -domains exercise table- and database-separated HMAC-SHA-256 equality indexes, -random append-row locators, versioned logical records, and the canonical -database/schema/table/locator AEAD binding above. - -Opening that destination validates the complete physical schema and every row. -Fixtures cover future or disagreeing schemas, incomplete or duplicate -migrations, invalid inner logical keys and record versions, duplicate indexes, -cross-database/table/row transplantation, wrong keys, and locked-copy -inspection. The legacy `Store` entry points refuse a v2 destination before -running legacy schema statements. - -This slice does **not** convert any current user table or activate a mixed -format. The all-table logical mapping, backup/restore integration, sibling-file -fsync and atomic replacement, rollback-copy lifecycle, private history query -indexes, large-history benchmarks, remnant reduction, and supported-platform -qualification remain required before ADR-0027 can be accepted. +`kult-store` now uses one generic physical record table for all 25 sensitive +logical domains. SQLite sees only domain numbers, random or keyed locators, +domain-separated keyed indexes, insertion order, and sealed bytes. Opening +validates the exact physical schema, metadata ledger, every row's canonical +AEAD identity, decoded logical key, and authenticated secondary indexes. +Unknown record versions and any inner-key, locator, or index disagreement fail +closed. + +Pairwise and group history have keyed message-id and conversation indexes, +bounded opaque cursor pages, and exact indexed update/delete. Query-plan tests +verify the intended SQLite indexes, and locked-copy tests check that known +contact, group, message, media, and delivery identifiers do not occur in raw +schema or index columns. + +Opening a released legacy schema runs the sibling-database migration. Released +fixtures cover `v0.1.0`, `v0.2.0`, and `v0.3.0`; row and total-count limits, +logical-key and referential checks, a source fingerprint, 256-row transactions, +a sealed restart checkpoint, count validation, full reopen validation, explicit +space checks, file and directory sync, a rollback copy, atomic replacement, +and phase-by-phase restart tests cover the replacement lifecycle. Restore uses +the same sibling-file discipline. Encrypted backups carry validated logical +records only, never a database id, opaque index, SQLite page, or wrapped v2 row +ciphertext. + +Bounded maintenance enables SQLite secure-delete behavior, incremental vacuum, +and WAL checkpoint/truncation where supported. Its result explicitly reports +that forensic erasure is not guaranteed. The 100,000- and 1,000,000-message +qualification gates enforce budgets for migration, unlock, page lookup, exact +edit/delete, peak resident-memory growth, and database growth. + +The exact implementation, measurements, replacement phase matrix, and +remaining filesystem evidence are recorded in +[33: Opaque Store Qualification](../33-opaque-store-qualification.md). +Independent storage review, physical power-loss testing, platform backup +exclusion, Windows owner-only ACL enforcement, and real macOS, Windows, Android, +and iOS filesystem qualification remain open. Those gaps limit platform and +forensic claims; they do not leave the legacy plaintext-index schema active. diff --git a/docs/adr/README.md b/docs/adr/README.md index 9461487..d61c7b5 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -33,7 +33,7 @@ product promise merely because the file exists. | [0024](0024-account-authorized-linked-devices.md) | Accepted Alpha; security-limited | Confirmed linking, per-device cryptography, sync, exact-id exclusion, and recovery; permanent-revocation claim withdrawn | | [0025](0025-optional-freenet-carrier.md) | Proposed | Optional epoch-scoped Freenet store-and-forward carrier and metadata boundary | | [0026](0026-revocable-device-authority.md) | Proposed (P0) | Required offline account-root recovery and quorum-authorized manifests replacing ADR-0024 authority before stable | -| [0027](0027-opaque-indexed-store.md) | Proposed; inactive v2 foundation implemented | Versioned keyed indexes, row-bound sealing, transactional migration, and honest local deletion limits | +| [0027](0027-opaque-indexed-store.md) | Accepted; implemented | Versioned keyed indexes, row-bound sealing, transactional migration, and honest local deletion limits | | [0028](0028-atomic-protocol-commits.md) | Proposed | Transactional ratchet, handshake, group-chain, queue, replay, and deferred-inbox state transitions | | [0029](0029-recipient-authenticated-groups.md) | Proposed | Per-recipient origin authentication for sender-key group messages and author-sensitive state | | [0030](0030-first-contact-admission.md) | Proposed | Bounded public introductions, provisional message requests, blocking, and group-invite consent | diff --git a/scripts/local-release-matrix.sh b/scripts/local-release-matrix.sh index af9ff85..8341acb 100755 --- a/scripts/local-release-matrix.sh +++ b/scripts/local-release-matrix.sh @@ -27,6 +27,7 @@ run_in "$root" cargo clippy --workspace --all-targets --all-features run_in "$root" cargo test --workspace --all-features run_in "$root" cargo build -p kult-crypto -p kult-protocol --no-default-features run_in "$root" cargo deny check +run_in "$root" scripts/store-scale-gate.sh desktop="$root/apps/desktop/src-tauri" run_in "$desktop" cargo fmt --all -- --check diff --git a/scripts/store-scale-gate.sh b/scripts/store-scale-gate.sh new file mode 100755 index 0000000..f64d624 --- /dev/null +++ b/scripts/store-scale-gate.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +for messages in 100000 1000000; do + printf '\n==> opaque store scale gate: %s messages\n' "$messages" + ( + cd "$root" + KOMMS_STORE_BENCH_MESSAGES="$messages" \ + cargo test --release -p kult-store \ + scale_bench::opaque_store_scale_budget -- \ + --exact --ignored --nocapture --test-threads=1 + ) +done