Read the write-ahead log: Database.AccessMode

README.md

@@ -95,6 +95,21 @@ print(result.stringValue)
 
 ## FAQ
 
+### Recent messages are missing
+
+Messages keeps `chat.db` in WAL mode:
+a new message is committed to `chat.db-wal`
+and only copied into `chat.db` at the next checkpoint,
+every few MB of writes — often hours later.
+`Database(path:)` reads the log when `chat.db-wal` and `chat.db-shm` are readable
+(`mode: .automatic`, the default)
+and otherwise falls back to SQLite's `immutable=1`,
+which sees the main file alone.
+`db.accessMode` tells which one you got.
+Pass `mode: .live` to get an error instead of stale data,
+or `mode: .immutable` when a sandbox grant covers only `chat.db`
+and staleness is acceptable.
+
 ### "Database Disk Image is Malformed"
 
 If you get the error message

Sources/iMessage/Database.swift

@@ -8,6 +8,10 @@ private let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.sel
 public final class Database {
     var db: OpaquePointer?
 
+    /// How the database file was opened: ``AccessMode/live`` or ``AccessMode/immutable``
+    /// (``AccessMode/automatic`` resolves to one of the two).
+    public let accessMode: AccessMode
+
     /// Defines flags used to open a SQLite database connection.
     public struct Flags: OptionSet, Sendable, Hashable {
         /// The underlying SQLite bitmask value.
@@ -58,29 +62,77 @@ public final class Database {
         case queryError(String)
     }
 
+    /// How the database file is read.
+    ///
+    /// Messages keeps `chat.db` in WAL mode:
+    /// a committed message first lands in `chat.db-wal`
+    /// and only reaches `chat.db` itself
+    /// when SQLite checkpoints the log (every few MB of writes, which can take hours).
+    /// Whether that log is read decides how current the results are.
+    public enum AccessMode: Sendable, Hashable {
+        /// Reads the database together with its write-ahead log (`chat.db-wal`)
+        /// and shared-memory index (`chat.db-shm`),
+        /// so every committed message is visible.
+        /// Requires read access to those companion files as well;
+        /// a read-only grant on the directory is enough.
+        case live
+        /// Opens with SQLite's `immutable=1`:
+        /// the main file only, no locks,
+        /// and the write-ahead log is ignored.
+        /// Works when nothing but `chat.db` itself is readable
+        /// (a sandboxed app whose user-selected grant covers that single file),
+        /// but messages written since the last checkpoint stay invisible until the next one.
+        case immutable
+        /// ``live`` when the companion files can be read,
+        /// ``immutable`` otherwise.
+        case automatic
+    }
+
     /// Backward-compatible alias for a message fetch request.
     public typealias MessageFetchRequest = FetchRequest<Message>
     /// Backward-compatible alias for a chat fetch request.
     public typealias ChatFetchRequest = FetchRequest<Chat>
 
-    private init(
-        _ filename: String,
-        flags: Flags = .default
-    ) throws {
-        if sqlite3_open_v2(filename, &db, flags.rawValue, nil) != SQLITE_OK {
-            throw Error.failedToOpen(String(cString: sqlite3_errmsg(db)))
+    private init(handle: OpaquePointer?, accessMode: AccessMode) {
+        self.db = handle
+        self.accessMode = accessMode
+    }
+
+    /// Opens a SQLite handle,
+    /// closing it again when SQLite reports a failure.
+    private static func open(_ filename: String, flags: Flags) throws -> OpaquePointer? {
+        var handle: OpaquePointer?
+        guard sqlite3_open_v2(filename, &handle, flags.rawValue, nil) == SQLITE_OK else {
+            let message = String(cString: sqlite3_errmsg(handle))
+            sqlite3_close(handle)
+            throw Error.failedToOpen(message)
         }
+        // A live connection shares locks with Messages;
+        // wait briefly instead of failing.
+        sqlite3_busy_timeout(handle, 1000)
+        return handle
+    }
+
+    /// Whether a first read succeeds.
+    /// On a WAL-mode file this is the moment SQLite opens the `-wal` and `-shm` companions,
+    /// so it fails when they are unreadable.
+    private static func canRead(_ handle: OpaquePointer?) -> Bool {
+        return sqlite3_exec(handle, "SELECT 1 FROM sqlite_master LIMIT 1", nil, nil, nil)
+            == SQLITE_OK
     }
 
     /// Opens the Messages database at a path.
     ///
     /// When `path` is `nil`, this initializer uses the default
     /// `~/Library/Messages/chat.db` location.
     ///
-    /// - Parameter path: An optional absolute database path.
+    /// - Parameters:
+    ///   - path: An optional absolute database path.
+    ///   - mode: How the file is read; see ``AccessMode``.
+    ///     Defaults to ``AccessMode/automatic``.
     /// - Throws: ``Error/databaseNotFound`` when the file does not exist,
     ///   or ``Error/failedToOpen(_:)`` when SQLite fails to open it.
-    public convenience init(path: String? = nil) throws {
+    public convenience init(path: String? = nil, mode: AccessMode = .automatic) throws {
         let resolvedPath: String
         if let path = path {
             resolvedPath = path
@@ -92,24 +144,48 @@ public final class Database {
             throw Error.databaseNotFound
         }
 
-        let dbURI = "file:\(resolvedPath)?immutable=1&mode=ro"
-        try self.init(dbURI, flags: [.readOnly, .uri])
+        let liveURI = "file:\(resolvedPath)?mode=ro"
+        let immutableURI = "file:\(resolvedPath)?immutable=1&mode=ro"
+
+        switch mode {
+        case .live:
+            self.init(handle: try Database.open(liveURI, flags: .default), accessMode: .live)
+        case .immutable:
+            self.init(
+                handle: try Database.open(immutableURI, flags: .default),
+                accessMode: .immutable
+            )
+        case .automatic:
+            let handle = try Database.open(liveURI, flags: .default)
+            if Database.canRead(handle) {
+                self.init(handle: handle, accessMode: .live)
+            } else {
+                sqlite3_close(handle)
+                self.init(
+                    handle: try Database.open(immutableURI, flags: .default),
+                    accessMode: .immutable
+                )
+            }
+        }
     }
 
     /// Creates an in-memory database handle for tests and temporary data.
     ///
     /// - Returns: A database opened at SQLite's `:memory:` location.
     /// - Throws: ``Error/failedToOpen(_:)`` when SQLite cannot create the database.
     public static func inMemory() throws -> Database {
-        return try Database(":memory:", flags: [.readWrite, .create])
+        return Database(
+            handle: try open(":memory:", flags: [.readWrite, .create]),
+            accessMode: .live
+        )
     }
 
     deinit {
         sqlite3_close(db)
     }
 
     // Remove transaction from execute
-    private func execute<T>(
+    func execute<T>(
         _ query: String,
         parameters: [any Bindable] = [],
         transform: (OpaquePointer) throws -> T?
@@ -128,10 +204,19 @@ public final class Database {
         }
 
         var results: [T] = []
-        while sqlite3_step(statement) == SQLITE_ROW {
+        var status = sqlite3_step(statement)
+        while status == SQLITE_ROW {
             if let result = try transform(statement) {
                 results.append(result)
             }
+            status = sqlite3_step(statement)
+        }
+
+        // Anything other than a clean end of results is an error,
+        // including `SQLITE_BUSY` when a live connection loses a race with Messages;
+        // returning what was read so far would silently truncate the results.
+        guard status == SQLITE_DONE else {
+            throw Error.queryError(String(cString: sqlite3_errmsg(db)))
         }
 
         return results
@@ -747,7 +832,7 @@ private extension SortOrder {
     }
 }
 
-private protocol Bindable {
+protocol Bindable {
     func bind(to statement: OpaquePointer, at index: Int32)
 }
 
@@ -758,19 +843,19 @@ extension String: Bindable {
 }
 
 extension Double: Bindable {
-    fileprivate func bind(to statement: OpaquePointer, at index: Int32) {
+    func bind(to statement: OpaquePointer, at index: Int32) {
         sqlite3_bind_double(statement, index, self)
     }
 }
 
 extension Int32: Bindable {
-    fileprivate func bind(to statement: OpaquePointer, at index: Int32) {
+    func bind(to statement: OpaquePointer, at index: Int32) {
         sqlite3_bind_int(statement, index, self)
     }
 }
 
 extension Int64: Bindable {
-    fileprivate func bind(to statement: OpaquePointer, at index: Int32) {
+    func bind(to statement: OpaquePointer, at index: Int32) {
         sqlite3_bind_int64(statement, index, self)
     }
 }

Tests/iMessageTests/AccessModeTests.swift

@@ -0,0 +1,120 @@
+import Foundation
+import SQLite3
+import Testing
+
+@testable import iMessage
+
+/// A WAL-mode database with one row checkpointed into the main file
+/// and a second row still in the write-ahead log.
+/// The writer stays open
+/// so nothing checkpoints it.
+private final class WALFixture {
+    let path: String
+    private var writer: OpaquePointer?
+
+    init() throws {
+        path =
+            FileManager.default.temporaryDirectory
+            .appendingPathComponent("madrid-\(UUID().uuidString).db").path
+        guard
+            sqlite3_open_v2(path, &writer, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nil)
+                == SQLITE_OK
+        else {
+            throw Database.Error.failedToOpen(String(cString: sqlite3_errmsg(writer)))
+        }
+        try execute("PRAGMA journal_mode=WAL")
+        try execute("CREATE TABLE t(x INTEGER)")
+        try execute("INSERT INTO t VALUES (1)")
+        try execute("PRAGMA wal_checkpoint(TRUNCATE)")
+        try execute("INSERT INTO t VALUES (2)")
+    }
+
+    deinit {
+        sqlite3_close(writer)
+        for suffix in ["", "-wal", "-shm"] {
+            try? FileManager.default.removeItem(atPath: path + suffix)
+        }
+    }
+
+    private func execute(_ sql: String) throws {
+        var error: UnsafeMutablePointer<CChar>?
+        if sqlite3_exec(writer, sql, nil, nil, &error) != SQLITE_OK {
+            let message = String(cString: error!)
+            sqlite3_free(error)
+            throw Database.Error.queryError(message)
+        }
+    }
+}
+
+private func rowCount(_ db: Database) throws -> Int {
+    var statement: OpaquePointer?
+    guard sqlite3_prepare_v2(db.db, "SELECT count(*) FROM t", -1, &statement, nil) == SQLITE_OK,
+        sqlite3_step(statement) == SQLITE_ROW
+    else {
+        throw Database.Error.queryError(String(cString: sqlite3_errmsg(db.db)))
+    }
+    defer { sqlite3_finalize(statement) }
+    return Int(sqlite3_column_int64(statement, 0))
+}
+
+@Suite(.serialized)
+struct AccessModeTests {
+    @Test
+    func liveReadsTheWriteAheadLog() throws {
+        let fixture = try WALFixture()
+        let db = try Database(path: fixture.path, mode: .live)
+        #expect(db.accessMode == .live)
+        #expect(try rowCount(db) == 2)
+    }
+
+    @Test
+    func immutableStopsAtTheLastCheckpoint() throws {
+        let fixture = try WALFixture()
+        let db = try Database(path: fixture.path, mode: .immutable)
+        #expect(db.accessMode == .immutable)
+        #expect(try rowCount(db) == 1)
+    }
+
+    @Test
+    func liveReadsThroughReadOnlyCompanions() throws {
+        // A sandboxed reader gets the Messages folder read-only:
+        // SQLite must cope with a `-shm` it cannot write to.
+        let fixture = try WALFixture()
+        let attributes = [FileAttributeKey.posixPermissions: 0o444]
+        try FileManager.default.setAttributes(attributes, ofItemAtPath: fixture.path + "-wal")
+        try FileManager.default.setAttributes(attributes, ofItemAtPath: fixture.path + "-shm")
+        defer {
+            let restore = [FileAttributeKey.posixPermissions: 0o644]
+            try? FileManager.default.setAttributes(restore, ofItemAtPath: fixture.path + "-wal")
+            try? FileManager.default.setAttributes(restore, ofItemAtPath: fixture.path + "-shm")
+        }
+        let db = try Database(path: fixture.path, mode: .live)
+        #expect(db.accessMode == .live)
+        #expect(try rowCount(db) == 2)
+    }
+
+    @Test
+    func automaticPrefersLiveWhenTheLogIsReadable() throws {
+        let fixture = try WALFixture()
+        let db = try Database(path: fixture.path)
+        #expect(db.accessMode == .live)
+        #expect(try rowCount(db) == 2)
+    }
+
+    @Test
+    func automaticFallsBackWhenTheLogIsUnreadable() throws {
+        let fixture = try WALFixture()
+        // Take the companions away from the reader the way a single-file grant does.
+        let attributes = [FileAttributeKey.posixPermissions: 0]
+        try FileManager.default.setAttributes(attributes, ofItemAtPath: fixture.path + "-wal")
+        try FileManager.default.setAttributes(attributes, ofItemAtPath: fixture.path + "-shm")
+        defer {
+            let restore = [FileAttributeKey.posixPermissions: 0o644]
+            try? FileManager.default.setAttributes(restore, ofItemAtPath: fixture.path + "-wal")
+            try? FileManager.default.setAttributes(restore, ofItemAtPath: fixture.path + "-shm")
+        }
+        let db = try Database(path: fixture.path)
+        #expect(db.accessMode == .immutable)
+        #expect(try rowCount(db) == 1)
+    }
+}

Tests/iMessageTests/DatabaseTests.swift

@@ -322,4 +322,20 @@ struct DatabaseTests {
             // Expected.
         }
     }
+
+    @Test("Query that fails mid-iteration throws instead of returning partial results")
+    func stepErrorMidIterationThrows() throws {
+        let db = try Database.inMemory()
+        // The second row overflows `abs`, so `sqlite3_step` returns an error
+        // after one row has already been produced.
+        let query = """
+            SELECT CASE WHEN x = 2 THEN abs(-9223372036854775808) ELSE x END
+            FROM (SELECT 1 AS x UNION ALL SELECT 2)
+            """
+        #expect(throws: Database.Error.self) {
+            try db.execute(query) { statement in
+                sqlite3_column_int64(statement, 0)
+            }
+        }
+    }
 }