From 59d6fb286b37a49cd67e67624345b406fb7950f6 Mon Sep 17 00:00:00 2001 From: Daniel Lyons <72824209+DandyLyons@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:34:33 -0500 Subject: [PATCH] Adopt GRDB with system SQLite for native indexing --- .github/workflows/sqlite-index.yml | 34 ++++ AGENTS.md | 5 +- Dockerfile.sqlite-index | 8 + IntegrationTests/SQLiteIndexSmoke/main.swift | 17 ++ Package.resolved | 11 +- Package.swift | 19 +++ .../SQLiteIndexDatabase.swift | 71 ++++++++ .../SQLiteIndexTests.swift | 79 +++++++++ docs/sqlite-index-packaging.md | 157 ++++++++++++++++++ docs/sqlite-library-evaluation.md | 82 +++++++++ scripts/measure-sqlite.py | 105 ++++++++++++ 11 files changed, 585 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/sqlite-index.yml create mode 100644 Dockerfile.sqlite-index create mode 100644 IntegrationTests/SQLiteIndexSmoke/main.swift create mode 100644 Sources/MarkdownUtilitiesIndex/SQLiteIndexDatabase.swift create mode 100644 Tests/MarkdownUtilitiesIndexTests/SQLiteIndexTests.swift create mode 100644 docs/sqlite-index-packaging.md create mode 100644 docs/sqlite-library-evaluation.md create mode 100644 scripts/measure-sqlite.py diff --git a/.github/workflows/sqlite-index.yml b/.github/workflows/sqlite-index.yml new file mode 100644 index 0000000..698baeb --- /dev/null +++ b/.github/workflows/sqlite-index.yml @@ -0,0 +1,34 @@ +name: Native SQLite +on: + pull_request: + paths: + - 'Package.swift' + - 'Package.resolved' + - 'Sources/MarkdownUtilitiesIndex/**' + - 'Tests/MarkdownUtilitiesIndexTests/**' + - 'IntegrationTests/SQLiteIndexSmoke/**' + - 'scripts/*sqlite*' + - 'Dockerfile.sqlite-index' + - '.github/workflows/sqlite-index.yml' + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read +jobs: + native: + strategy: + matrix: + os: [macos-26, ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v5 + - name: Linux capabilities, failure semantics, measurements, and linkage + if: runner.os == 'Linux' + run: docker build --file Dockerfile.sqlite-index --tag md-utils-sqlite . + - name: macOS capabilities and failure semantics + if: runner.os == 'macOS' + run: | + swift run -c release SQLiteIndexSmoke + swift test --filter MarkdownUtilitiesIndexTests + python3 scripts/measure-sqlite.py diff --git a/AGENTS.md b/AGENTS.md index a651c83..df69110 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,18 +12,19 @@ md-utils is a Swift package for parsing and manipulating Markdown files. It cons ## Project Brief - **Language**: Swift 6.2+ +- **Native indexing prototype**: `MarkdownUtilitiesIndex` uses upstream GRDB with system SQLite through SwiftPM; JSON, expression indexes, and FTS5 are probed before opening index files. Core and WASM have no GRDB/SQLite dependency. See `docs/sqlite-index-packaging.md` for runtime requirements, measurements, and the optional SwiftPM bundled-runtime fallback. Xcode-based database integration is rejected. - **Frameworks/Libraries**: Foundation, MarkdownSyntax, PathKit, Yams, swift-toml, JMESPath, DynamicJSON, JSONSchema.swift, swift-argument-parser, Noora (interactive CLI authoring), Rainbow, Hummingbird 2, Swift Logging - **parsing**: Any code that involves parsing text must use the `Parsing` library like the rest of the codebase. - **Package Manager / Build Tool**: Swift Package Manager - **Executable Targets**: `md-utils`, `md-utils-server` -- **Library Targets**: `MarkdownUtilitiesCore`, `MarkdownUtilities` +- **Library Targets**: `MarkdownUtilitiesCore`, `MarkdownUtilities`, `MarkdownUtilitiesServer`, `MarkdownUtilitiesIndex` - **Test Framework**: Swift Testing, not XCTest - **Build Command**: `swift build` - **Test Command**: `swift test`; native Linux server route smoke test with `swift run MarkdownUtilitiesServerLinuxSmoke` - **Formatter/Linter**: No dedicated formatter or linter is configured in-package - **Documentation**: README.md, AGENTS.md, docs/*.md, generated CLI help, and bundled Agent Skill docs - **Security**: Avoid unsafe optional force unwraps; treat filesystem and YAML/TOML/JSON parsing failures as user-visible errors -- **CI/Coverage**: Schema publication, Pages, WebAssembly, and native Linux server workflows are configured; local Linux server verification uses `Dockerfile.server-linux`; no coverage command is documented +- **CI/Coverage**: Schema publication, Pages, WebAssembly, native Linux server, and native SQLite workflows are configured; local verification uses `Dockerfile.server-linux` and `Dockerfile.sqlite-index`; no coverage command is documented ## Requirements diff --git a/Dockerfile.sqlite-index b/Dockerfile.sqlite-index new file mode 100644 index 0000000..48d31ce --- /dev/null +++ b/Dockerfile.sqlite-index @@ -0,0 +1,8 @@ +FROM swift:6.2-noble +RUN apt-get update && apt-get install -y --no-install-recommends python3 libsqlite3-dev binutils && rm -rf /var/lib/apt/lists/* +WORKDIR /workspace/ +COPY Package.swift Package.resolved ./ +RUN swift package resolve +COPY . . +RUN swift run -c release SQLiteIndexSmoke +RUN python3 scripts/measure-sqlite.py diff --git a/IntegrationTests/SQLiteIndexSmoke/main.swift b/IntegrationTests/SQLiteIndexSmoke/main.swift new file mode 100644 index 0000000..0456587 --- /dev/null +++ b/IntegrationTests/SQLiteIndexSmoke/main.swift @@ -0,0 +1,17 @@ +import MarkdownUtilitiesIndex +import GRDBSQLite + +// A consumer can use the system C API in the same executable as the GRDB facade. +// Both resolve to the system SQLite library; there is no second bundled runtime. +enum SmokeError: Error { case systemConnectionFailed } +var consumer: OpaquePointer? +guard sqlite3_open(":memory:", &consumer) == SQLITE_OK, let connection = consumer else { + if let consumer { sqlite3_close(consumer) } + throw SmokeError.systemConnectionFailed +} +defer { sqlite3_close(connection) } +guard sqlite3_exec(connection, "CREATE TABLE consumer(value TEXT)", nil, nil, nil) == SQLITE_OK else { + throw SmokeError.systemConnectionFailed +} +try SQLiteIndexDatabase.checkCapabilities() +print("System SQLite \(SQLiteIndexDatabase.sqliteVersion): GRDB JSON, expression indexes, FTS5, and C consumer passed") diff --git a/Package.resolved b/Package.resolved index fd37ecc..c52b2cf 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "8b6c758041e90f4d0c6e670693fc633eb217279671ecb76313413ce9e66e58db", + "originHash" : "112f08ab25a7178f8f34efd04a8f0ce2293183ee18e7f00c62814783f95abf36", "pins" : [ { "identity" : "async-http-client", @@ -10,6 +10,15 @@ "version" : "1.36.0" } }, + { + "identity" : "grdb.swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/groue/GRDB.swift.git", + "state" : { + "revision" : "b83108d10f42680d78f23fe4d4d80fc88dab3212", + "version" : "7.11.1" + } + }, { "identity" : "hummingbird", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index 285ce21..6f4c553 100644 --- a/Package.swift +++ b/Package.swift @@ -9,6 +9,7 @@ let package = Package( .macOS(.v13), .iOS(.v16), .tvOS(.v16), .watchOS(.v9), .macCatalyst(.v16), ], products: [ + .library(name: "MarkdownUtilitiesIndex", targets: ["MarkdownUtilitiesIndex"]), .library( name: "MarkdownUtilitiesCore", targets: ["MarkdownUtilitiesCore"] @@ -31,6 +32,7 @@ let package = Package( ), ], dependencies: [ + .package(url: "https://github.com/groue/GRDB.swift.git", from: "7.11.1"), .package(url: "https://github.com/hebertialmeida/MarkdownSyntax", from: "1.3.0"), .package(url: "https://github.com/pointfreeco/swift-parsing.git", from: "0.14.1"), .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.6.1"), @@ -52,6 +54,23 @@ let package = Package( .package(url: "https://github.com/apple/swift-docc-plugin.git", from: "1.4.0"), ], targets: [ + // Native indexing is opt-in; Core, WASM, and existing commands do not link SQLite. + .target( + name: "MarkdownUtilitiesIndex", + dependencies: [ + .product(name: "GRDB", package: "GRDB.swift"), + .product(name: "GRDBSQLite", package: "GRDB.swift"), + ] + ), + .testTarget(name: "MarkdownUtilitiesIndexTests", dependencies: ["MarkdownUtilitiesIndex"]), + .executableTarget( + name: "SQLiteIndexSmoke", + dependencies: [ + "MarkdownUtilitiesIndex", + .product(name: "GRDBSQLite", package: "GRDB.swift"), + ], + path: "IntegrationTests/SQLiteIndexSmoke/" + ), // MARK: MarkdownUtilitiesCore .target( name: "MarkdownUtilitiesCore", diff --git a/Sources/MarkdownUtilitiesIndex/SQLiteIndexDatabase.swift b/Sources/MarkdownUtilitiesIndex/SQLiteIndexDatabase.swift new file mode 100644 index 0000000..df5a49f --- /dev/null +++ b/Sources/MarkdownUtilitiesIndex/SQLiteIndexDatabase.swift @@ -0,0 +1,71 @@ +import GRDB +import GRDBSQLite + +/// A native SQLite connection reserved for the rebuildable file index. +/// GRDB serializes access to the connection. Indexing operations will be added here. +public final class SQLiteIndexDatabase { + private let databaseQueue: DatabaseQueue + + /// The system SQLite runtime used by GRDB, not a bundled version. + public static var sqliteVersion: String { String(cString: sqlite3_libversion()) } + + /// Checks the linked runtime in memory before creating or opening an index file. + public convenience init(path: String) throws { + try self.init(path: path, probe: Self.checkCapabilities) + } + + internal init(path: String, probe: () throws -> Void) throws { + try probe() + do { + databaseQueue = try DatabaseQueue(path: path) + } catch { + throw SQLiteIndexError(message: "Cannot open SQLite index at \(path): \(error). Check the parent directory and permissions.") + } + } + + /// Exercises JSON queries, JSON expression indexes, and FTS5 reads and writes. + /// No index file is touched by this probe. + public static func checkCapabilities() throws { + try checkCapabilities(probes: capabilityProbes) + } + + internal static func checkCapabilities(probes: [(String, String)]) throws { + do { + let queue = try DatabaseQueue() + try queue.write { database in + for (capability, sql) in probes { + do { + try database.execute(sql: sql) + } catch { + throw SQLiteIndexError(message: "System SQLite \(sqliteVersion) failed the required \(capability) check: \(error). Use a supported OS or distribution SQLite package with JSON, expression indexes, and FTS5 enabled. Updating GRDB alone does not update SQLite. The index was not opened.") + } + } + } + } catch let error as SQLiteIndexError { + throw error + } catch { + throw SQLiteIndexError(message: "Cannot probe system SQLite \(sqliteVersion): \(error). Check the OS or distribution SQLite installation. The index was not opened.") + } + } + + internal static let capabilityProbes: [(String, String)] = [ + ("JSON queries and expression indexes", """ + CREATE TABLE records(document TEXT); + CREATE INDEX record_title ON records(json_extract(document, '$.title')); + INSERT INTO records VALUES ('{"title":"hello"}'); + CREATE TABLE assertions(ok INTEGER CHECK(ok = 1)); + INSERT INTO assertions SELECT count(*) = 1 FROM records + INDEXED BY record_title WHERE json_extract(document, '$.title') = 'hello'; + """), + ("FTS5", """ + CREATE VIRTUAL TABLE search USING fts5(body); + INSERT INTO search VALUES ('hello indexing'); + INSERT INTO assertions SELECT count(*) = 1 FROM search WHERE search MATCH 'indexing'; + """), + ] +} + +public struct SQLiteIndexError: Error, CustomStringConvertible, Sendable { + public let message: String + public var description: String { message } +} diff --git a/Tests/MarkdownUtilitiesIndexTests/SQLiteIndexTests.swift b/Tests/MarkdownUtilitiesIndexTests/SQLiteIndexTests.swift new file mode 100644 index 0000000..01e30d4 --- /dev/null +++ b/Tests/MarkdownUtilitiesIndexTests/SQLiteIndexTests.swift @@ -0,0 +1,79 @@ +import Foundation +import Testing +@testable import MarkdownUtilitiesIndex + +private func temporaryFile() throws -> URL { + let directory = URL(fileURLWithPath: FileManager.default.currentDirectoryPath) + .appendingPathComponent("tmp/sqlite-tests/", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + return directory.appendingPathComponent(UUID().uuidString) +} + +@Test func `system runtime supports all required capabilities`() throws { + #expect(!SQLiteIndexDatabase.sqliteVersion.isEmpty) + try SQLiteIndexDatabase.checkCapabilities() +} + +@Test func `capability failure identifies runtime and remediation`() throws { + do { + try SQLiteIndexDatabase.checkCapabilities(probes: [("JSON", "SELECT missing_json_function('{}')")]) + Issue.record("Expected capability failure") + } catch let error as SQLiteIndexError { + #expect(error.description.contains("JSON")) + #expect(error.description.contains(SQLiteIndexDatabase.sqliteVersion)) + #expect(error.description.contains("OS or distribution SQLite package")) + #expect(error.description.contains("Updating GRDB alone does not update SQLite")) + #expect(error.description.contains("index was not opened")) + } +} + +@Test func `successful probe permits opening an index file`() throws { + let file = try temporaryFile() + defer { try? FileManager.default.removeItem(at: file) } + let database = try SQLiteIndexDatabase(path: file.path) + withExtendedLifetime(database) { + #expect(FileManager.default.fileExists(atPath: file.path)) + } +} + +@Test func `failed FTS probe does not create an index`() throws { + let path = try temporaryFile().path + #expect(throws: SQLiteIndexError.self) { + _ = try SQLiteIndexDatabase(path: path) { + try SQLiteIndexDatabase.checkCapabilities(probes: [("FTS5", "CREATE VIRTUAL TABLE unavailable USING missing_fts5(body)")]) + } + } + #expect(!FileManager.default.fileExists(atPath: path)) +} + +@Test func `failed JSON probe preserves an existing file`() throws { + let file = try temporaryFile() + let original = Data("existing index".utf8) + try original.write(to: file) + defer { try? FileManager.default.removeItem(at: file) } + #expect(throws: SQLiteIndexError.self) { + _ = try SQLiteIndexDatabase(path: file.path) { + try SQLiteIndexDatabase.checkCapabilities(probes: [("JSON", "SELECT missing_json_function('{}')")]) + } + } + #expect(try Data(contentsOf: file) == original) +} + +@Test func `invalid parent reports an actionable open error`() throws { + let path = try temporaryFile().appendingPathComponent("index.sqlite").path + do { + _ = try SQLiteIndexDatabase(path: path) + Issue.record("Expected file-open failure") + } catch let error as SQLiteIndexError { + #expect(error.description.contains(path)) + #expect(error.description.contains("parent directory and permissions")) + } +} + +@Test func `probe rejects incorrect query results`() { + #expect(throws: SQLiteIndexError.self) { + try SQLiteIndexDatabase.checkCapabilities(probes: [ + ("result assertions", "CREATE TABLE assertions(ok INTEGER CHECK(ok = 1)); INSERT INTO assertions VALUES (0);") + ]) + } +} diff --git a/docs/sqlite-index-packaging.md b/docs/sqlite-index-packaging.md new file mode 100644 index 0000000..1f7ad0f --- /dev/null +++ b/docs/sqlite-index-packaging.md @@ -0,0 +1,157 @@ +# Native SQLite packaging + +Issue #137 establishes the database dependency for the indexing phase of #93. +`MarkdownUtilitiesIndex` is an opt-in native library using upstream GRDB through +SwiftPM and the operating system's SQLite runtime. It provides a checked +connection foundation; indexing, queries, refresh, and server integration remain +in #138–#141. Files remain authoritative. + +## Decision and alternatives + +**Selected: option 1, GRDB with system SQLite.** No amalgamation, custom SQLite +build, or second runtime is shipped. SwiftPM currently resolves GRDB 7.11.1. +GRDB owns connection serialization and provides statement, transaction, and +migration infrastructure for later indexing work. + +**Fallback: option 3, a source-based SwiftPM adaptation of GRDB with bundled +SQLite.** Reconsider only if system SQLite cannot meet supported deployment +requirements. That would require a maintainable C-module/package adaptation, +matched compile flags, consumer symbol-isolation checks, and macOS/Linux tests. +It is not implemented or selected automatically on a failed probe. + +**Rejected: option 2, the Xcode-based custom SQLite framework workflow.** Database +integration must preserve ordinary SwiftPM builds and Linux support. GRDB's +[documented custom-build workflow](https://github.com/groue/GRDB.swift/blob/v7.11.1/Documentation/CustomSQLiteBuilds.md) +does not provide that integration. + +The earlier bundled prototype and C bridge have been removed. Their measurements +are not estimates of GRDB overhead. See [the library evaluation](sqlite-library-evaluation.md) +for the rationale and the separate, deferred StructuredQueries decision. + +## Runtime requirements and supported targets + +The linked SQLite must support JSON functions, JSON expression indexes, and +FTS5. A version string or GRDB Swift compilation flag does not establish those +capabilities. Modern SQLite includes JSON unless omitted; FTS5 must be enabled +in the runtime build. [SQLite JSON](https://sqlite.org/json1.html#compiling_in_json_support), +[SQLite FTS5](https://sqlite.org/fts5.html). + +| Platform or product | Policy | +| --- | --- | +| Native index, macOS 13+ | Use system SQLite with mandatory capability probes. Local validation uses macOS 27 arm64, not every older OS. | +| Native index, Linux | Use distribution SQLite. Ubuntu Noble with Swift 6.2 is the reference container; CI covers x86_64 and local Docker validation covers arm64. Other distributions require validation. | +| iOS 16+, tvOS 16+, watchOS 9+, Mac Catalyst 16+ | Package minimums are unchanged. The prototype has not validated these SDKs/runtimes; do not infer index support from the manifest alone. | +| Windows, Android | Not supported by this prototype. | +| `MarkdownUtilitiesCore` and Core WASM smoke | No dependency path to GRDB or SQLite. | +| `MarkdownUtilities`, CLI, server | Do not yet depend on the index target; existing commands do not run database probes. | + +Linux source builds require SQLite development headers and a linker library +(Ubuntu/Debian: `libsqlite3-dev`); deployment requires the runtime library +(`libsqlite3-0` on Ubuntu/Debian). The measurement script additionally uses +Python 3 and binutils. A missing shared library is an installation error that +can prevent a linked executable from launching; capability checks cover a +present runtime with missing or incompatible features. + +GRDB's [package](https://github.com/groue/GRDB.swift/blob/v7.11.1/Package.swift) +uses system SQLite. Upstream describes Linux support as contributor-maintained; +this repository owns its Linux validation. Consumer smoke tests exercise direct +SQLite C access and GRDB in one process. Binary inspection verifies dynamic +`libsqlite3` linkage and absence of embedded `sqlite3_*` definitions. This checks +ordinary system-runtime coexistence, not arbitrary third-party bundled builds +or process-global SQLite reconfiguration. + +## Capability and failure contract + +`SQLiteIndexDatabase(path:)` first opens an in-memory GRDB `DatabaseQueue`. +It creates and queries a JSON expression index and creates, writes, and searches +an FTS5 table. SQL CHECK assertions validate query results. Only after those +operations succeed does it open the requested index file. No migration or schema +change is attempted before the probe succeeds. + +On failure, `SQLiteIndexError` identifies the system runtime version, failed +capability, SQLite error, and OS/distribution remediation. Updating GRDB alone +does not update SQLite. There is no silent reduced-functionality mode or bundled +fallback. Tests use real failing statements and check incorrect query results, +preservation of existing file bytes, and absence of newly created files. Open +errors identify the path and suggest checking parent directories and permissions. +Existing non-index commands remain usable when index capabilities are missing. + +All future index mutations must use the checked initializer. Do not open a +separate unchecked GRDB connection before probing. Keep public schemas based on +JSON text, standard expression indexes, and FTS5, without GRDB-only SQL functions +or custom extensions. External tools use their own SQLite runtime and must +independently support those facilities; installing md-utils does not upgrade +GUI browsers or the `sqlite3` command. + +## Verification and measurements + +Run from the repository root: + +```sh +swift run SQLiteIndexSmoke +swift test --filter MarkdownUtilitiesIndexTests +python3 scripts/measure-sqlite.py +docker build --file Dockerfile.sqlite-index --tag md-utils-sqlite . +scripts/build-wasm.sh +``` + +The measurement script creates a fresh package under `tmp/`, copies the actual +index/smoke/test sources, and uses the root lockfile's GRDB version. It does not +reuse build products. This isolates native overhead from the CLI/server graph; +the Docker recipe also builds the real root-package smoke target. + +The baseline is a Swift executable calling system SQLite directly. The index +smoke adds GRDB and the checked connection facade. Download/resolution time is +reported separately. The baseline compiles first; GRDB has not been compiled +when the index release build starts. Both original release and stripped sizes +are reported; both executables are run, followed by linkage/symbol checks. Tests +run in the isolated package as well as the host's full suite. + +Measurements are single-run observations, not statistical benchmarks or final +CLI/server size predictions. No indexing throughput claim is made. Reports are +saved as `report.json` in each workspace and printed in CI. Existing CLI/server +products acquire no GRDB linkage until they explicitly depend on the index. +Package resolution still sees GRDB even when building unrelated products. + +Local measurements (September 16, 2026, GRDB 7.11.1): macOS 27 arm64 uses +system SQLite 3.54.0 and Swift 6.4 with its matching SDK 27; Ubuntu Noble arm64 +in Docker uses system SQLite 3.45.1 and Swift 6.2.4. + +| Measurement | macOS | Linux | +| --- | --- | --- | +| Dependency resolution, separately timed | 5.861 s | 3.163 s | +| Clean direct-SQLite baseline build | 8.398 s | 0.333 s | +| First GRDB/index release build | 25.193 s | 20.411 s | +| Warm no-op index build | 1.032 s | 0.173 s | +| Baseline release / stripped size | 51,928 / 50,768 bytes | 78,640 / 69,456 bytes | +| Index smoke release / stripped size | 5,385,008 / 2,751,352 bytes | 12,668,768 / 2,993,656 bytes | + +The stripped executable deltas are 2,700,584 bytes on macOS and 2,924,200 bytes +on Linux. Timings include SwiftPM planning; the first baseline also warms SDK +module caches. The index build still compiles GRDB from scratch. Linux validation +used downloaded source repository caches after a network fetch stalled, not +host build products. Both the root release smoke and the isolated measurements, +seven tests, stripped executables, and linkage checks passed on Linux. + +The standalone Swift 6.3.1 compiler crashed in release IR generation for GRDB's +`NSUUID.fromDatabaseValue` when paired with the installed SDK 27 beta. Debug +builds and all 1,350 host tests passed with that compiler. Repeating the release +measurement using the matching SDK compiler succeeded, including the seven +focused tests and both stripped executables. This is a toolchain compatibility +limitation, not a GRDB source patch or adoption of the rejected Xcode-framework +workflow: all builds still use SwiftPM. Use a matching compiler/SDK for macOS +release builds and revalidate new combinations. + +## Maintenance + +Before each md-utils release, review GRDB releases and applicable SQLite fixes. +Update the GRDB constraint through SwiftPM, allow SwiftPM to generate +`Package.resolved`, and rerun native tests, release measurements, linkage checks, +and the Core WASM smoke. Never manually edit the lockfile. + +SQLite security/correctness updates come from the OS or distribution. Document +unsupported environments and known runtime defects even if basic probes pass; +these probes are not a complete SQLite conformance test. macOS remediation may +require an OS update. Linux users need a supported distribution/runtime package, +with appropriate development headers for source builds. If those requirements +become unacceptable, explicitly evaluate option 3. diff --git a/docs/sqlite-library-evaluation.md b/docs/sqlite-library-evaluation.md new file mode 100644 index 0000000..683bd73 --- /dev/null +++ b/docs/sqlite-library-evaluation.md @@ -0,0 +1,82 @@ +# GRDB and StructuredQueries evaluation + +Reviewed September 16, 2026 for the indexing phase of #93. The selected approach +is now upstream GRDB with system SQLite through SwiftPM (option 1). The GRDB +integration and measurements are described in [SQLite packaging](sqlite-index-packaging.md). +StructuredQueries remains deferred; its evaluation below is a source review, +not an integration benchmark. + +| Candidate | Benefit here | Cost | Decision | +| --- | --- | --- | --- | +| GRDB | Bindings, decoding, transactions, migrations, serialized access and pools | System-runtime prerequisites, Linux validation, Swift compilation | Adopted through upstream SwiftPM packaging | +| StructuredQueries | Typed, composable internal queries | Macros, driver integration, another API to maintain | Revisit after schema and queries stabilize | +| SQLiteData | GRDB/StructuredQueries integration and observation | Broader application-state dependency graph | Do not adopt solely to connect the other two | + +## GRDB decision + +GRDB's infrastructure maps to transactional document/assessment/FTS updates in +#138 and consistent reads in #139/#141. It also supplies its own query builder. +Its observation features do not replace filesystem watching/reconciliation in +#140. The prior C bridge was only a capability prototype; retaining it would +have meant implementing production statement lifetimes, bindings, decoding, +concurrency, cancellation, and error propagation ourselves. +[GRDB README](https://github.com/groue/GRDB.swift/blob/v7.11.1/README.md). + +GRDB v7.11.1 requires Swift 6.1 and uses a system SQLite target. It has no normal +external Swift package dependency (its documentation plugin is conditional). +Linux snapshot APIs are disabled by its manifest. Upstream describes Linux +support as contributor-maintained, so this repository owns Linux validation. +[Manifest](https://github.com/groue/GRDB.swift/blob/v7.11.1/Package.swift). + +Option 1 accepts OS/distribution SQLite version and capability variation, guarded +by executable probes and documented prerequisites. No bundled runtime remains. +If this becomes unsuitable, option 3 is a fallback: adapt GRDB's C module and +SwiftPM packaging to a controlled bundle, retaining symbol isolation and Linux +support. That work is not selected automatically or included here. Option 2, +GRDB's Xcode custom-framework workflow, is explicitly rejected. +[Custom SQLite workflow](https://github.com/groue/GRDB.swift/blob/v7.11.1/Documentation/CustomSQLiteBuilds.md). + +## StructuredQueries assessment + +StructuredQueries builds typed SQL and requires a driver. Its documented custom +integration needs `QueryDecoder` and statement execution helpers; GRDB is not +technically mandatory. Adding it does not replace connection/transaction work. +[Driver integration](https://github.com/pointfreeco/swift-structured-queries/blob/0.39.2/Sources/StructuredQueriesCore/Documentation.docc/Articles/Integration.md). + +The best fit is the fixed internal schema: files, documents, memberships, +assessments, and joins. User-defined metadata, JSON paths, configured views, and +arbitrary SQL in #139 remain runtime concerns. A typed builder cannot prove user +SQL is read-only or guarantee expression-index usage. We still need read-only +enforcement and query-plan tests; raw SQL escape hatches reduce compile-time +protection for dynamic pieces. + +Release 0.39.2 has a Swift 6.1-specific manifest compatible with our toolchain +minimum despite its default manifest declaring Swift 6.4. Macro products use +SwiftSyntax; the basic core uses IssueReporting, with optional CasePaths/Tagged +traits. Test dependencies are not all runtime dependencies. Our lockfile already +contains SwiftSyntax, CasePaths, and xctest-dynamic-overlay: measure incremental +cost rather than charging the entire graph again. Macro compilation still +matters for an otherwise small native target. +[Versioned manifest](https://github.com/pointfreeco/swift-structured-queries/blob/0.39.2/Package%40swift-6.1.swift). + +The README mentions `StructuredQueriesGRDB`, but inspected SQLiteData manifests +expose `SQLiteData` and `SQLiteDataTestSupport`, not a separately selectable GRDB +adapter. Its main target also uses Sharing, Perception, Dependencies, and +ConcurrencyExtras. Verify the chosen release's products before assuming the +adapter can be consumed alone. +[Driver overview](https://github.com/pointfreeco/swift-structured-queries#database-drivers), +[SQLiteData manifest](https://github.com/pointfreeco/sqlite-data/blob/main/Package%40swift-6.1.swift). + +## Revisit during #139 + +Compare several actual queries implemented with bound SQL, GRDB's own builder, +and StructuredQueries. Include optional values, joins, dynamic JSON fields, and +expression-index plans. Measure clean builds, query-edit build latency, stripped +binary size, and representative refresh/query performance with identical SQLite +versions, schemas, and transaction boundaries. Use multiple runs and separate +dependency downloads and host macro compilation from target compilation. + +Adopt StructuredQueries only if examples show a meaningful safety/maintenance +gain that outweighs those costs. Keep it in the native indexing target; neither +GRDB nor StructuredQueries belongs in portable Core. There are no measured +StructuredQueries overhead figures yet. diff --git a/scripts/measure-sqlite.py b/scripts/measure-sqlite.py new file mode 100644 index 0000000..21e0b94 --- /dev/null +++ b/scripts/measure-sqlite.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Measure native GRDB overhead and test copied production sources in isolation. + +Every run uses a fresh project-local build directory. The generated package uses +the root lockfile's GRDB pin, but no unrelated CLI/server dependencies or builds. +""" +import json +from pathlib import Path +import platform +import shutil +import subprocess +import tempfile +import time + +ROOT = Path(__file__).resolve().parent.parent +pins = json.loads((ROOT / "Package.resolved").read_text())["pins"] +pin = next(pin for pin in pins if pin["identity"] == "grdb.swift") +version = pin["state"]["version"] +scratch = ROOT / "tmp/" +scratch.mkdir(exist_ok=True) +work = Path(tempfile.mkdtemp(prefix="grdb-measure-", dir=scratch)) +print(f"Measurement workspace: {work}/", flush=True) + +# Copy source, never reuse another build directory. Also run the actual index tests. +for source, destination in [ + ("Sources/MarkdownUtilitiesIndex/", "Sources/MarkdownUtilitiesIndex/"), + ("IntegrationTests/SQLiteIndexSmoke/", "Sources/SQLiteIndexSmoke/"), + ("Tests/MarkdownUtilitiesIndexTests/", "Tests/MarkdownUtilitiesIndexTests/"), +]: + shutil.copytree(ROOT / source, work / destination) +(work / "Sources/Baseline/").mkdir(parents=True) +(work / "Sources/Baseline/main.swift").write_text( + 'import GRDBSQLite\nprint("System SQLite \\(String(cString: sqlite3_libversion()))")\n' +) +(work / "Package.swift").write_text('''// swift-tools-version: 6.2 +import PackageDescription +let package = Package( + name: "SQLiteMeasurement", + platforms: [.macOS(.v13)], + dependencies: [.package(url: "https://github.com/groue/GRDB.swift.git", exact: "VERSION")], + targets: [ + .target(name: "MarkdownUtilitiesIndex", dependencies: [ + .product(name: "GRDB", package: "GRDB.swift"), + .product(name: "GRDBSQLite", package: "GRDB.swift"), + ]), + .executableTarget(name: "SQLiteIndexSmoke", dependencies: [ + "MarkdownUtilitiesIndex", .product(name: "GRDBSQLite", package: "GRDB.swift"), + ]), + .executableTarget(name: "Baseline", dependencies: [ + .product(name: "GRDBSQLite", package: "GRDB.swift"), + ]), + .testTarget(name: "MarkdownUtilitiesIndexTests", dependencies: ["MarkdownUtilitiesIndex"]), + ] +) +'''.replace("VERSION", version)) + +def run(arguments): + print("+", " ".join(map(str, arguments)), flush=True) + start = time.perf_counter() + subprocess.run(list(map(str, arguments)), cwd=work, check=True) + return time.perf_counter() - start + +run(["swift", "--version"]) +resolve_seconds = run(["swift", "package", "resolve"]) +baseline_seconds = run(["swift", "build", "-c", "release", "--product", "Baseline"]) +grdb_seconds = run(["swift", "build", "-c", "release", "--product", "SQLiteIndexSmoke"]) +warm_seconds = run(["swift", "build", "-c", "release", "--product", "SQLiteIndexSmoke"]) +bin_path = Path(subprocess.check_output( + ["swift", "build", "-c", "release", "--show-bin-path"], cwd=work, text=True +).strip()) +run([bin_path / "Baseline"]) +run([bin_path / "SQLiteIndexSmoke"]) +run(["swift", "test", "--filter", "MarkdownUtilitiesIndexTests"]) + +sizes = {} +for name in ("Baseline", "SQLiteIndexSmoke"): + executable = bin_path / name + stripped = work / f"{name}-stripped" + shutil.copy2(executable, stripped) + run(["strip", stripped]) + run([stripped]) + sizes[name] = {"release_bytes": executable.stat().st_size, "stripped_bytes": stripped.stat().st_size} + +# Verify dynamic system SQLite linkage and reject accidentally embedded SQLite definitions. +if platform.system() == "Darwin": + linkage = subprocess.check_output(["otool", "-L", str(bin_path / "SQLiteIndexSmoke")], text=True) + symbols = subprocess.check_output(["nm", "-gU", str(bin_path / "SQLiteIndexSmoke")], text=True) +else: + linkage = subprocess.check_output(["ldd", str(bin_path / "SQLiteIndexSmoke")], text=True) + symbols = subprocess.check_output(["nm", "-g", "--defined-only", str(bin_path / "SQLiteIndexSmoke")], text=True) +print(linkage, flush=True) +if "libsqlite3" not in linkage: + raise SystemExit("Expected dynamic system SQLite linkage") +if any(line.split()[-1].lstrip("_").startswith("sqlite3_") for line in symbols.splitlines() if line.split()): + raise SystemExit("Unexpected embedded SQLite definition") + +report = { + "platform": platform.platform(), "grdb_version": version, + "dependency_resolution_seconds": round(resolve_seconds, 3), + "clean_baseline_seconds": round(baseline_seconds, 3), + "clean_grdb_seconds": round(grdb_seconds, 3), + "warm_noop_seconds": round(warm_seconds, 3), "sizes": sizes, +} +(work / "report.json").write_text(json.dumps(report, indent=2) + "\n") +print(json.dumps(report, indent=2), flush=True)