From 26346a8d00a5dd9ef123aea761b10917ebbd4bbd Mon Sep 17 00:00:00 2001 From: patp Date: Tue, 1 Sep 2026 17:39:15 -0400 Subject: [PATCH 1/4] Read the write-ahead log: Database.AccessMode Messages keeps chat.db in WAL mode, and Database(path:) opened it with immutable=1, which makes SQLite ignore chat.db-wal. Every message committed since the last checkpoint was therefore invisible; checkpoints happen every few MB of writes, so the newest messages lagged by hours and then appeared in batches. Database.AccessMode chooses how the file is read: .live (mode=ro, the -wal/-shm companions are read; a read-only grant on the directory is enough), .immutable (the previous behaviour, for a grant that covers chat.db alone) and .automatic (default: live, falling back to immutable when the first read cannot open the companions). init(path:) is unchanged for existing callers, accessMode reports the outcome, and a 1 s busy timeout covers the locks a live connection shares with Messages. Tests build a WAL-mode fixture with one checkpointed row and one still in the log and check the three modes, including the fallback. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0187kUP2mjhwMfJkwRbP5DG7 --- README.md | 15 ++++ Sources/iMessage/Database.swift | 91 +++++++++++++++++--- Tests/iMessageTests/AccessModeTests.swift | 100 ++++++++++++++++++++++ 3 files changed, 195 insertions(+), 11 deletions(-) create mode 100644 Tests/iMessageTests/AccessModeTests.swift diff --git a/README.md b/README.md index 873aa69..59b4153 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,21 @@ print(result.stringValue) ## FAQ +### Recent messages are missing + +Messages keeps `chat.db` in WAL mode: +a new message is committed to `chat.db-wal` +and only copied into `chat.db` at the next checkpoint, +every few MB of writes — often hours later. +`Database(path:)` reads the log when `chat.db-wal` and `chat.db-shm` are readable +(`mode: .automatic`, the default) +and otherwise falls back to SQLite's `immutable=1`, +which sees the main file alone. +`db.accessMode` tells which one you got. +Pass `mode: .live` to get an error instead of stale data, +or `mode: .immutable` when a sandbox grant covers only `chat.db` +and staleness is acceptable. + ### "Database Disk Image is Malformed" If you get the error message diff --git a/Sources/iMessage/Database.swift b/Sources/iMessage/Database.swift index 44175df..8e275c8 100644 --- a/Sources/iMessage/Database.swift +++ b/Sources/iMessage/Database.swift @@ -8,6 +8,10 @@ private let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.sel public final class Database { var db: OpaquePointer? + /// How the database file was opened: ``AccessMode/live`` or ``AccessMode/immutable`` + /// (``AccessMode/automatic`` resolves to one of the two). + public let accessMode: AccessMode + /// Defines flags used to open a SQLite database connection. public struct Flags: OptionSet, Sendable, Hashable { /// The underlying SQLite bitmask value. @@ -58,18 +62,56 @@ public final class Database { case queryError(String) } + /// How the database file is read. + /// + /// Messages keeps `chat.db` in WAL mode: a committed message first lands in + /// `chat.db-wal` and only reaches `chat.db` itself when SQLite checkpoints the + /// log (every few MB of writes, which can take hours). Whether that log is read + /// decides how current the results are. + public enum AccessMode: Sendable, Hashable { + /// Reads the database together with its write-ahead log (`chat.db-wal`) and + /// shared-memory index (`chat.db-shm`), so every committed message is visible. + /// Requires read access to those companion files as well; a read-only grant + /// on the directory is enough. + case live + /// Opens with SQLite's `immutable=1`: the main file only, no locks, and the + /// write-ahead log is ignored. Works when nothing but `chat.db` itself is + /// readable (a sandboxed app whose user-selected grant covers that single + /// file), but messages written since the last checkpoint stay invisible until + /// the next one. + case immutable + /// ``live`` when the companion files can be read, ``immutable`` otherwise. + case automatic + } + /// Backward-compatible alias for a message fetch request. public typealias MessageFetchRequest = FetchRequest /// Backward-compatible alias for a chat fetch request. public typealias ChatFetchRequest = FetchRequest - private init( - _ filename: String, - flags: Flags = .default - ) throws { - if sqlite3_open_v2(filename, &db, flags.rawValue, nil) != SQLITE_OK { - throw Error.failedToOpen(String(cString: sqlite3_errmsg(db))) + private init(handle: OpaquePointer?, accessMode: AccessMode) { + self.db = handle + self.accessMode = accessMode + } + + /// Opens a SQLite handle, closing it again when SQLite reports a failure. + private static func open(_ filename: String, flags: Flags) throws -> OpaquePointer? { + var handle: OpaquePointer? + guard sqlite3_open_v2(filename, &handle, flags.rawValue, nil) == SQLITE_OK else { + let message = String(cString: sqlite3_errmsg(handle)) + sqlite3_close(handle) + throw Error.failedToOpen(message) } + // A live connection shares locks with Messages; wait briefly instead of failing. + sqlite3_busy_timeout(handle, 1000) + return handle + } + + /// Whether a first read succeeds. On a WAL-mode file this is the moment SQLite + /// opens the `-wal` and `-shm` companions, so it fails when they are unreadable. + private static func canRead(_ handle: OpaquePointer?) -> Bool { + return sqlite3_exec(handle, "SELECT 1 FROM sqlite_master LIMIT 1", nil, nil, nil) + == SQLITE_OK } /// Opens the Messages database at a path. @@ -77,10 +119,13 @@ public final class Database { /// When `path` is `nil`, this initializer uses the default /// `~/Library/Messages/chat.db` location. /// - /// - Parameter path: An optional absolute database path. + /// - Parameters: + /// - path: An optional absolute database path. + /// - mode: How the file is read; see ``AccessMode``. Defaults to + /// ``AccessMode/automatic``. /// - Throws: ``Error/databaseNotFound`` when the file does not exist, /// or ``Error/failedToOpen(_:)`` when SQLite fails to open it. - public convenience init(path: String? = nil) throws { + public convenience init(path: String? = nil, mode: AccessMode = .automatic) throws { let resolvedPath: String if let path = path { resolvedPath = path @@ -92,8 +137,29 @@ public final class Database { throw Error.databaseNotFound } - let dbURI = "file:\(resolvedPath)?immutable=1&mode=ro" - try self.init(dbURI, flags: [.readOnly, .uri]) + let liveURI = "file:\(resolvedPath)?mode=ro" + let immutableURI = "file:\(resolvedPath)?immutable=1&mode=ro" + + switch mode { + case .live: + self.init(handle: try Database.open(liveURI, flags: .default), accessMode: .live) + case .immutable: + self.init( + handle: try Database.open(immutableURI, flags: .default), + accessMode: .immutable + ) + case .automatic: + let handle = try Database.open(liveURI, flags: .default) + if Database.canRead(handle) { + self.init(handle: handle, accessMode: .live) + } else { + sqlite3_close(handle) + self.init( + handle: try Database.open(immutableURI, flags: .default), + accessMode: .immutable + ) + } + } } /// Creates an in-memory database handle for tests and temporary data. @@ -101,7 +167,10 @@ public final class Database { /// - Returns: A database opened at SQLite's `:memory:` location. /// - Throws: ``Error/failedToOpen(_:)`` when SQLite cannot create the database. public static func inMemory() throws -> Database { - return try Database(":memory:", flags: [.readWrite, .create]) + return Database( + handle: try open(":memory:", flags: [.readWrite, .create]), + accessMode: .live + ) } deinit { diff --git a/Tests/iMessageTests/AccessModeTests.swift b/Tests/iMessageTests/AccessModeTests.swift new file mode 100644 index 0000000..005f441 --- /dev/null +++ b/Tests/iMessageTests/AccessModeTests.swift @@ -0,0 +1,100 @@ +import Foundation +import SQLite3 +import Testing + +@testable import iMessage + +/// A WAL-mode database with one row checkpointed into the main file and a second row +/// still in the write-ahead log. The writer stays open so nothing checkpoints it. +private final class WALFixture { + let path: String + private var writer: OpaquePointer? + + init() throws { + path = + FileManager.default.temporaryDirectory + .appendingPathComponent("madrid-\(UUID().uuidString).db").path + guard + sqlite3_open_v2(path, &writer, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nil) + == SQLITE_OK + else { + throw Database.Error.failedToOpen(String(cString: sqlite3_errmsg(writer))) + } + try execute("PRAGMA journal_mode=WAL") + try execute("CREATE TABLE t(x INTEGER)") + try execute("INSERT INTO t VALUES (1)") + try execute("PRAGMA wal_checkpoint(TRUNCATE)") + try execute("INSERT INTO t VALUES (2)") + } + + deinit { + sqlite3_close(writer) + for suffix in ["", "-wal", "-shm"] { + try? FileManager.default.removeItem(atPath: path + suffix) + } + } + + private func execute(_ sql: String) throws { + var error: UnsafeMutablePointer? + if sqlite3_exec(writer, sql, nil, nil, &error) != SQLITE_OK { + let message = String(cString: error!) + sqlite3_free(error) + throw Database.Error.queryError(message) + } + } +} + +private func rowCount(_ db: Database) throws -> Int { + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db.db, "SELECT count(*) FROM t", -1, &statement, nil) == SQLITE_OK, + sqlite3_step(statement) == SQLITE_ROW + else { + throw Database.Error.queryError(String(cString: sqlite3_errmsg(db.db))) + } + defer { sqlite3_finalize(statement) } + return Int(sqlite3_column_int64(statement, 0)) +} + +@Suite(.serialized) +struct AccessModeTests { + @Test + func liveReadsTheWriteAheadLog() throws { + let fixture = try WALFixture() + let db = try Database(path: fixture.path, mode: .live) + #expect(db.accessMode == .live) + #expect(try rowCount(db) == 2) + } + + @Test + func immutableStopsAtTheLastCheckpoint() throws { + let fixture = try WALFixture() + let db = try Database(path: fixture.path, mode: .immutable) + #expect(db.accessMode == .immutable) + #expect(try rowCount(db) == 1) + } + + @Test + func automaticPrefersLiveWhenTheLogIsReadable() throws { + let fixture = try WALFixture() + let db = try Database(path: fixture.path) + #expect(db.accessMode == .live) + #expect(try rowCount(db) == 2) + } + + @Test + func automaticFallsBackWhenTheLogIsUnreadable() throws { + let fixture = try WALFixture() + // Take the companions away from the reader the way a single-file grant does. + let attributes = [FileAttributeKey.posixPermissions: 0] + try FileManager.default.setAttributes(attributes, ofItemAtPath: fixture.path + "-wal") + try FileManager.default.setAttributes(attributes, ofItemAtPath: fixture.path + "-shm") + defer { + let restore = [FileAttributeKey.posixPermissions: 0o644] + try? FileManager.default.setAttributes(restore, ofItemAtPath: fixture.path + "-wal") + try? FileManager.default.setAttributes(restore, ofItemAtPath: fixture.path + "-shm") + } + let db = try Database(path: fixture.path) + #expect(db.accessMode == .immutable) + #expect(try rowCount(db) == 1) + } +} From c9fb272d476d6035f785dbca998242b236648e80 Mon Sep 17 00:00:00 2001 From: patp Date: Tue, 1 Sep 2026 17:42:23 -0400 Subject: [PATCH 2/4] Test live mode against read-only -wal/-shm companions A sandboxed app that was granted the Messages folder reads it read-only, so SQLite has to work with a shared-memory index it cannot write to. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0187kUP2mjhwMfJkwRbP5DG7 --- Tests/iMessageTests/AccessModeTests.swift | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Tests/iMessageTests/AccessModeTests.swift b/Tests/iMessageTests/AccessModeTests.swift index 005f441..910f65c 100644 --- a/Tests/iMessageTests/AccessModeTests.swift +++ b/Tests/iMessageTests/AccessModeTests.swift @@ -73,6 +73,24 @@ struct AccessModeTests { #expect(try rowCount(db) == 1) } + @Test + func liveReadsThroughReadOnlyCompanions() throws { + // A sandboxed reader gets the Messages folder read-only: SQLite must cope with a + // `-shm` it cannot write to. + let fixture = try WALFixture() + let attributes = [FileAttributeKey.posixPermissions: 0o444] + try FileManager.default.setAttributes(attributes, ofItemAtPath: fixture.path + "-wal") + try FileManager.default.setAttributes(attributes, ofItemAtPath: fixture.path + "-shm") + defer { + let restore = [FileAttributeKey.posixPermissions: 0o644] + try? FileManager.default.setAttributes(restore, ofItemAtPath: fixture.path + "-wal") + try? FileManager.default.setAttributes(restore, ofItemAtPath: fixture.path + "-shm") + } + let db = try Database(path: fixture.path, mode: .live) + #expect(db.accessMode == .live) + #expect(try rowCount(db) == 2) + } + @Test func automaticPrefersLiveWhenTheLogIsReadable() throws { let fixture = try WALFixture() From 9abe5d495ec3cb02e338fe6e2dcfe3bfa904b400 Mon Sep 17 00:00:00 2001 From: mattt Date: Wed, 2 Sep 2026 10:25:26 -0700 Subject: [PATCH 3/4] Reflow comments with semantic line breaks --- Sources/iMessage/Database.swift | 47 +++++++++++++---------- Tests/iMessageTests/AccessModeTests.swift | 10 +++-- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/Sources/iMessage/Database.swift b/Sources/iMessage/Database.swift index 8e275c8..eef0275 100644 --- a/Sources/iMessage/Database.swift +++ b/Sources/iMessage/Database.swift @@ -64,23 +64,27 @@ public final class Database { /// How the database file is read. /// - /// Messages keeps `chat.db` in WAL mode: a committed message first lands in - /// `chat.db-wal` and only reaches `chat.db` itself when SQLite checkpoints the - /// log (every few MB of writes, which can take hours). Whether that log is read - /// decides how current the results are. + /// Messages keeps `chat.db` in WAL mode: + /// a committed message first lands in `chat.db-wal` + /// and only reaches `chat.db` itself + /// when SQLite checkpoints the log (every few MB of writes, which can take hours). + /// Whether that log is read decides how current the results are. public enum AccessMode: Sendable, Hashable { - /// Reads the database together with its write-ahead log (`chat.db-wal`) and - /// shared-memory index (`chat.db-shm`), so every committed message is visible. - /// Requires read access to those companion files as well; a read-only grant - /// on the directory is enough. + /// Reads the database together with its write-ahead log (`chat.db-wal`) + /// and shared-memory index (`chat.db-shm`), + /// so every committed message is visible. + /// Requires read access to those companion files as well; + /// a read-only grant on the directory is enough. case live - /// Opens with SQLite's `immutable=1`: the main file only, no locks, and the - /// write-ahead log is ignored. Works when nothing but `chat.db` itself is - /// readable (a sandboxed app whose user-selected grant covers that single - /// file), but messages written since the last checkpoint stay invisible until - /// the next one. + /// Opens with SQLite's `immutable=1`: + /// the main file only, no locks, + /// and the write-ahead log is ignored. + /// Works when nothing but `chat.db` itself is readable + /// (a sandboxed app whose user-selected grant covers that single file), + /// but messages written since the last checkpoint stay invisible until the next one. case immutable - /// ``live`` when the companion files can be read, ``immutable`` otherwise. + /// ``live`` when the companion files can be read, + /// ``immutable`` otherwise. case automatic } @@ -94,7 +98,8 @@ public final class Database { self.accessMode = accessMode } - /// Opens a SQLite handle, closing it again when SQLite reports a failure. + /// Opens a SQLite handle, + /// closing it again when SQLite reports a failure. private static func open(_ filename: String, flags: Flags) throws -> OpaquePointer? { var handle: OpaquePointer? guard sqlite3_open_v2(filename, &handle, flags.rawValue, nil) == SQLITE_OK else { @@ -102,13 +107,15 @@ public final class Database { sqlite3_close(handle) throw Error.failedToOpen(message) } - // A live connection shares locks with Messages; wait briefly instead of failing. + // A live connection shares locks with Messages; + // wait briefly instead of failing. sqlite3_busy_timeout(handle, 1000) return handle } - /// Whether a first read succeeds. On a WAL-mode file this is the moment SQLite - /// opens the `-wal` and `-shm` companions, so it fails when they are unreadable. + /// Whether a first read succeeds. + /// On a WAL-mode file this is the moment SQLite opens the `-wal` and `-shm` companions, + /// so it fails when they are unreadable. private static func canRead(_ handle: OpaquePointer?) -> Bool { return sqlite3_exec(handle, "SELECT 1 FROM sqlite_master LIMIT 1", nil, nil, nil) == SQLITE_OK @@ -121,8 +128,8 @@ public final class Database { /// /// - Parameters: /// - path: An optional absolute database path. - /// - mode: How the file is read; see ``AccessMode``. Defaults to - /// ``AccessMode/automatic``. + /// - mode: How the file is read; see ``AccessMode``. + /// Defaults to ``AccessMode/automatic``. /// - Throws: ``Error/databaseNotFound`` when the file does not exist, /// or ``Error/failedToOpen(_:)`` when SQLite fails to open it. public convenience init(path: String? = nil, mode: AccessMode = .automatic) throws { diff --git a/Tests/iMessageTests/AccessModeTests.swift b/Tests/iMessageTests/AccessModeTests.swift index 910f65c..4c2848e 100644 --- a/Tests/iMessageTests/AccessModeTests.swift +++ b/Tests/iMessageTests/AccessModeTests.swift @@ -4,8 +4,10 @@ import Testing @testable import iMessage -/// A WAL-mode database with one row checkpointed into the main file and a second row -/// still in the write-ahead log. The writer stays open so nothing checkpoints it. +/// A WAL-mode database with one row checkpointed into the main file +/// and a second row still in the write-ahead log. +/// The writer stays open +/// so nothing checkpoints it. private final class WALFixture { let path: String private var writer: OpaquePointer? @@ -75,8 +77,8 @@ struct AccessModeTests { @Test func liveReadsThroughReadOnlyCompanions() throws { - // A sandboxed reader gets the Messages folder read-only: SQLite must cope with a - // `-shm` it cannot write to. + // A sandboxed reader gets the Messages folder read-only: + // SQLite must cope with a `-shm` it cannot write to. let fixture = try WALFixture() let attributes = [FileAttributeKey.posixPermissions: 0o444] try FileManager.default.setAttributes(attributes, ofItemAtPath: fixture.path + "-wal") From f740ecedb501b83411b06c307974aa1498d675ee Mon Sep 17 00:00:00 2001 From: mattt Date: Wed, 2 Sep 2026 10:31:35 -0700 Subject: [PATCH 4/4] Throw when a query ends on anything other than SQLITE_DONE The step loop treated every non-row status as a clean end of results, so a step error, or SQLITE_BUSY on a live connection that loses a race with Messages, silently returned partial data. Surface it as Error.queryError instead. Make execute and Bindable internal so the case can be tested directly. --- Sources/iMessage/Database.swift | 21 +++++++++++++++------ Tests/iMessageTests/DatabaseTests.swift | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/Sources/iMessage/Database.swift b/Sources/iMessage/Database.swift index eef0275..5280bd2 100644 --- a/Sources/iMessage/Database.swift +++ b/Sources/iMessage/Database.swift @@ -185,7 +185,7 @@ public final class Database { } // Remove transaction from execute - private func execute( + func execute( _ query: String, parameters: [any Bindable] = [], transform: (OpaquePointer) throws -> T? @@ -204,10 +204,19 @@ public final class Database { } var results: [T] = [] - while sqlite3_step(statement) == SQLITE_ROW { + var status = sqlite3_step(statement) + while status == SQLITE_ROW { if let result = try transform(statement) { results.append(result) } + status = sqlite3_step(statement) + } + + // Anything other than a clean end of results is an error, + // including `SQLITE_BUSY` when a live connection loses a race with Messages; + // returning what was read so far would silently truncate the results. + guard status == SQLITE_DONE else { + throw Error.queryError(String(cString: sqlite3_errmsg(db))) } return results @@ -823,7 +832,7 @@ private extension SortOrder { } } -private protocol Bindable { +protocol Bindable { func bind(to statement: OpaquePointer, at index: Int32) } @@ -834,19 +843,19 @@ extension String: Bindable { } extension Double: Bindable { - fileprivate func bind(to statement: OpaquePointer, at index: Int32) { + func bind(to statement: OpaquePointer, at index: Int32) { sqlite3_bind_double(statement, index, self) } } extension Int32: Bindable { - fileprivate func bind(to statement: OpaquePointer, at index: Int32) { + func bind(to statement: OpaquePointer, at index: Int32) { sqlite3_bind_int(statement, index, self) } } extension Int64: Bindable { - fileprivate func bind(to statement: OpaquePointer, at index: Int32) { + func bind(to statement: OpaquePointer, at index: Int32) { sqlite3_bind_int64(statement, index, self) } } diff --git a/Tests/iMessageTests/DatabaseTests.swift b/Tests/iMessageTests/DatabaseTests.swift index b346b6b..44fe3d6 100644 --- a/Tests/iMessageTests/DatabaseTests.swift +++ b/Tests/iMessageTests/DatabaseTests.swift @@ -322,4 +322,20 @@ struct DatabaseTests { // Expected. } } + + @Test("Query that fails mid-iteration throws instead of returning partial results") + func stepErrorMidIterationThrows() throws { + let db = try Database.inMemory() + // The second row overflows `abs`, so `sqlite3_step` returns an error + // after one row has already been produced. + let query = """ + SELECT CASE WHEN x = 2 THEN abs(-9223372036854775808) ELSE x END + FROM (SELECT 1 AS x UNION ALL SELECT 2) + """ + #expect(throws: Database.Error.self) { + try db.execute(query) { statement in + sqlite3_column_int64(statement, 0) + } + } + } }