Adopt GRDB with system SQLite for native indexing

.github/workflows/sqlite-index.yml

@@ -0,0 +1,34 @@
+name: Native SQLite
+on:
+  pull_request:
+    paths:
+      - 'Package.swift'
+      - 'Package.resolved'
+      - 'Sources/MarkdownUtilitiesIndex/**'
+      - 'Tests/MarkdownUtilitiesIndexTests/**'
+      - 'IntegrationTests/SQLiteIndexSmoke/**'
+      - 'scripts/*sqlite*'
+      - 'Dockerfile.sqlite-index'
+      - '.github/workflows/sqlite-index.yml'
+  push:
+    branches: [main]
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  native:
+    strategy:
+      matrix:
+        os: [macos-26, ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v5
+      - name: Linux capabilities, failure semantics, measurements, and linkage
+        if: runner.os == 'Linux'
+        run: docker build --file Dockerfile.sqlite-index --tag md-utils-sqlite .
+      - name: macOS capabilities and failure semantics
+        if: runner.os == 'macOS'
+        run: |
+          swift run -c release SQLiteIndexSmoke
+          swift test --filter MarkdownUtilitiesIndexTests
+          python3 scripts/measure-sqlite.py

AGENTS.md

@@ -12,18 +12,19 @@ md-utils is a Swift package for parsing and manipulating Markdown files. It cons
 ## Project Brief
 
 - **Language**: Swift 6.2+
+- **Native indexing prototype**: `MarkdownUtilitiesIndex` uses upstream GRDB with system SQLite through SwiftPM; JSON, expression indexes, and FTS5 are probed before opening index files. Core and WASM have no GRDB/SQLite dependency. See `docs/sqlite-index-packaging.md` for runtime requirements, measurements, and the optional SwiftPM bundled-runtime fallback. Xcode-based database integration is rejected.
 - **Frameworks/Libraries**: Foundation, MarkdownSyntax, PathKit, Yams, swift-toml, JMESPath, DynamicJSON, JSONSchema.swift, swift-argument-parser, Noora (interactive CLI authoring), Rainbow, Hummingbird 2, Swift Logging
   - **parsing**: Any code that involves parsing text must use the `Parsing` library like the rest of the codebase.
 - **Package Manager / Build Tool**: Swift Package Manager
 - **Executable Targets**: `md-utils`, `md-utils-server`
-- **Library Targets**: `MarkdownUtilitiesCore`, `MarkdownUtilities`
+- **Library Targets**: `MarkdownUtilitiesCore`, `MarkdownUtilities`, `MarkdownUtilitiesServer`, `MarkdownUtilitiesIndex`
 - **Test Framework**: Swift Testing, not XCTest
 - **Build Command**: `swift build`
 - **Test Command**: `swift test`; native Linux server route smoke test with `swift run MarkdownUtilitiesServerLinuxSmoke`
 - **Formatter/Linter**: No dedicated formatter or linter is configured in-package
 - **Documentation**: README.md, AGENTS.md, docs/*.md, generated CLI help, and bundled Agent Skill docs
 - **Security**: Avoid unsafe optional force unwraps; treat filesystem and YAML/TOML/JSON parsing failures as user-visible errors
-- **CI/Coverage**: Schema publication, Pages, WebAssembly, and native Linux server workflows are configured; local Linux server verification uses `Dockerfile.server-linux`; no coverage command is documented
+- **CI/Coverage**: Schema publication, Pages, WebAssembly, native Linux server, and native SQLite workflows are configured; local verification uses `Dockerfile.server-linux` and `Dockerfile.sqlite-index`; no coverage command is documented
 
 ## Requirements
 

Dockerfile.sqlite-index

@@ -0,0 +1,8 @@
+FROM swift:6.2-noble
+RUN apt-get update && apt-get install -y --no-install-recommends python3 libsqlite3-dev binutils && rm -rf /var/lib/apt/lists/*
+WORKDIR /workspace/
+COPY Package.swift Package.resolved ./
+RUN swift package resolve
+COPY . .
+RUN swift run -c release SQLiteIndexSmoke
+RUN python3 scripts/measure-sqlite.py

IntegrationTests/SQLiteIndexSmoke/main.swift

@@ -0,0 +1,17 @@
+import MarkdownUtilitiesIndex
+import GRDBSQLite
+
+// A consumer can use the system C API in the same executable as the GRDB facade.
+// Both resolve to the system SQLite library; there is no second bundled runtime.
+enum SmokeError: Error { case systemConnectionFailed }
+var consumer: OpaquePointer?
+guard sqlite3_open(":memory:", &consumer) == SQLITE_OK, let connection = consumer else {
+    if let consumer { sqlite3_close(consumer) }
+    throw SmokeError.systemConnectionFailed
+}
+defer { sqlite3_close(connection) }
+guard sqlite3_exec(connection, "CREATE TABLE consumer(value TEXT)", nil, nil, nil) == SQLITE_OK else {
+    throw SmokeError.systemConnectionFailed
+}
+try SQLiteIndexDatabase.checkCapabilities()
+print("System SQLite \(SQLiteIndexDatabase.sqliteVersion): GRDB JSON, expression indexes, FTS5, and C consumer passed")

Package.swift

@@ -9,6 +9,7 @@ let package = Package(
     .macOS(.v13), .iOS(.v16), .tvOS(.v16), .watchOS(.v9), .macCatalyst(.v16),
   ],
   products: [
+    .library(name: "MarkdownUtilitiesIndex", targets: ["MarkdownUtilitiesIndex"]),
     .library(
       name: "MarkdownUtilitiesCore",
       targets: ["MarkdownUtilitiesCore"]
@@ -31,6 +32,7 @@ let package = Package(
     ),
   ],
   dependencies: [
+    .package(url: "https://github.com/groue/GRDB.swift.git", from: "7.11.1"),
     .package(url: "https://github.com/hebertialmeida/MarkdownSyntax", from: "1.3.0"),
     .package(url: "https://github.com/pointfreeco/swift-parsing.git", from: "0.14.1"),
     .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.6.1"),
@@ -52,6 +54,23 @@ let package = Package(
     .package(url: "https://github.com/apple/swift-docc-plugin.git", from: "1.4.0"),
   ],
   targets: [
+    // Native indexing is opt-in; Core, WASM, and existing commands do not link SQLite.
+    .target(
+      name: "MarkdownUtilitiesIndex",
+      dependencies: [
+        .product(name: "GRDB", package: "GRDB.swift"),
+        .product(name: "GRDBSQLite", package: "GRDB.swift"),
+      ]
+    ),
+    .testTarget(name: "MarkdownUtilitiesIndexTests", dependencies: ["MarkdownUtilitiesIndex"]),
+    .executableTarget(
+      name: "SQLiteIndexSmoke",
+      dependencies: [
+        "MarkdownUtilitiesIndex",
+        .product(name: "GRDBSQLite", package: "GRDB.swift"),
+      ],
+      path: "IntegrationTests/SQLiteIndexSmoke/"
+    ),
     // MARK: MarkdownUtilitiesCore
     .target(
       name: "MarkdownUtilitiesCore",

Sources/MarkdownUtilitiesIndex/SQLiteIndexDatabase.swift

@@ -0,0 +1,71 @@
+import GRDB
+import GRDBSQLite
+
+/// A native SQLite connection reserved for the rebuildable file index.
+/// GRDB serializes access to the connection. Indexing operations will be added here.
+public final class SQLiteIndexDatabase {
+    private let databaseQueue: DatabaseQueue
+
+    /// The system SQLite runtime used by GRDB, not a bundled version.
+    public static var sqliteVersion: String { String(cString: sqlite3_libversion()) }
+
+    /// Checks the linked runtime in memory before creating or opening an index file.
+    public convenience init(path: String) throws {
+        try self.init(path: path, probe: Self.checkCapabilities)
+    }
+
+    internal init(path: String, probe: () throws -> Void) throws {
+        try probe()
+        do {
+            databaseQueue = try DatabaseQueue(path: path)
+        } catch {
+            throw SQLiteIndexError(message: "Cannot open SQLite index at \(path): \(error). Check the parent directory and permissions.")
+        }
+    }
+
+    /// Exercises JSON queries, JSON expression indexes, and FTS5 reads and writes.
+    /// No index file is touched by this probe.
+    public static func checkCapabilities() throws {
+        try checkCapabilities(probes: capabilityProbes)
+    }
+
+    internal static func checkCapabilities(probes: [(String, String)]) throws {
+        do {
+            let queue = try DatabaseQueue()
+            try queue.write { database in
+                for (capability, sql) in probes {
+                    do {
+                        try database.execute(sql: sql)
+                    } catch {
+                        throw SQLiteIndexError(message: "System SQLite \(sqliteVersion) failed the required \(capability) check: \(error). Use a supported OS or distribution SQLite package with JSON, expression indexes, and FTS5 enabled. Updating GRDB alone does not update SQLite. The index was not opened.")
+                    }
+                }
+            }
+        } catch let error as SQLiteIndexError {
+            throw error
+        } catch {
+            throw SQLiteIndexError(message: "Cannot probe system SQLite \(sqliteVersion): \(error). Check the OS or distribution SQLite installation. The index was not opened.")
+        }
+    }
+
+    internal static let capabilityProbes: [(String, String)] = [
+        ("JSON queries and expression indexes", """
+        CREATE TABLE records(document TEXT);
+        CREATE INDEX record_title ON records(json_extract(document, '$.title'));
+        INSERT INTO records VALUES ('{"title":"hello"}');
+        CREATE TABLE assertions(ok INTEGER CHECK(ok = 1));
+        INSERT INTO assertions SELECT count(*) = 1 FROM records
+          INDEXED BY record_title WHERE json_extract(document, '$.title') = 'hello';
+        """),
+        ("FTS5", """
+        CREATE VIRTUAL TABLE search USING fts5(body);
+        INSERT INTO search VALUES ('hello indexing');
+        INSERT INTO assertions SELECT count(*) = 1 FROM search WHERE search MATCH 'indexing';
+        """),
+    ]
+}
+
+public struct SQLiteIndexError: Error, CustomStringConvertible, Sendable {
+    public let message: String
+    public var description: String { message }
+}

Tests/MarkdownUtilitiesIndexTests/SQLiteIndexTests.swift

@@ -0,0 +1,79 @@
+import Foundation
+import Testing
+@testable import MarkdownUtilitiesIndex
+
+private func temporaryFile() throws -> URL {
+    let directory = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
+        .appendingPathComponent("tmp/sqlite-tests/", isDirectory: true)
+    try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
+    return directory.appendingPathComponent(UUID().uuidString)
+}
+
+@Test func `system runtime supports all required capabilities`() throws {
+    #expect(!SQLiteIndexDatabase.sqliteVersion.isEmpty)
+    try SQLiteIndexDatabase.checkCapabilities()
+}
+
+@Test func `capability failure identifies runtime and remediation`() throws {
+    do {
+        try SQLiteIndexDatabase.checkCapabilities(probes: [("JSON", "SELECT missing_json_function('{}')")])
+        Issue.record("Expected capability failure")
+    } catch let error as SQLiteIndexError {
+        #expect(error.description.contains("JSON"))
+        #expect(error.description.contains(SQLiteIndexDatabase.sqliteVersion))
+        #expect(error.description.contains("OS or distribution SQLite package"))
+        #expect(error.description.contains("Updating GRDB alone does not update SQLite"))
+        #expect(error.description.contains("index was not opened"))
+    }
+}
+
+@Test func `successful probe permits opening an index file`() throws {
+    let file = try temporaryFile()
+    defer { try? FileManager.default.removeItem(at: file) }
+    let database = try SQLiteIndexDatabase(path: file.path)
+    withExtendedLifetime(database) {
+        #expect(FileManager.default.fileExists(atPath: file.path))
+    }
+}
+
+@Test func `failed FTS probe does not create an index`() throws {
+    let path = try temporaryFile().path
+    #expect(throws: SQLiteIndexError.self) {
+        _ = try SQLiteIndexDatabase(path: path) {
+            try SQLiteIndexDatabase.checkCapabilities(probes: [("FTS5", "CREATE VIRTUAL TABLE unavailable USING missing_fts5(body)")])
+        }
+    }
+    #expect(!FileManager.default.fileExists(atPath: path))
+}
+
+@Test func `failed JSON probe preserves an existing file`() throws {
+    let file = try temporaryFile()
+    let original = Data("existing index".utf8)
+    try original.write(to: file)
+    defer { try? FileManager.default.removeItem(at: file) }
+    #expect(throws: SQLiteIndexError.self) {
+        _ = try SQLiteIndexDatabase(path: file.path) {
+            try SQLiteIndexDatabase.checkCapabilities(probes: [("JSON", "SELECT missing_json_function('{}')")])
+        }
+    }
+    #expect(try Data(contentsOf: file) == original)
+}
+
+@Test func `invalid parent reports an actionable open error`() throws {
+    let path = try temporaryFile().appendingPathComponent("index.sqlite").path
+    do {
+        _ = try SQLiteIndexDatabase(path: path)
+        Issue.record("Expected file-open failure")
+    } catch let error as SQLiteIndexError {
+        #expect(error.description.contains(path))
+        #expect(error.description.contains("parent directory and permissions"))
+    }
+}
+
+@Test func `probe rejects incorrect query results`() {
+    #expect(throws: SQLiteIndexError.self) {
+        try SQLiteIndexDatabase.checkCapabilities(probes: [
+            ("result assertions", "CREATE TABLE assertions(ok INTEGER CHECK(ok = 1)); INSERT INTO assertions VALUES (0);")
+        ])
+    }
+}