From 7eaed064599bf5fd160f3976dd128e20695e08d5 Mon Sep 17 00:00:00 2001 From: 3x-haust Date: Mon, 15 Jun 2026 11:31:07 +0900 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20gitdb=20=EC=A0=84=EC=9A=A9=20API=20?= =?UTF-8?q?=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 56 ++--- biome.json | 1 + docs/API.md | 52 ++--- docs/ARCHITECTURE.md | 10 +- docs/BENCHMARKS.md | 34 +-- docs/BENCHMARK_BASELINE.json | 2 +- docs/MIGRATION.md | 13 +- docs/README.ko.md | 66 +++--- docs/RELEASE_NOTES.md | 18 +- examples/api-encrypted-reopen/index.mjs | 95 --------- examples/api-encrypted/index.mjs | 136 ++++++++---- examples/api-local/index.mjs | 69 ------ examples/api-plaintext/index.mjs | 105 +++++++++ package.json | 7 +- scripts/benchmark-gate.mjs | 21 +- scripts/benchmark-report.mjs | 25 +-- scripts/benchmark-scenarios.mjs | 90 ++++---- scripts/benchmark-utils.mjs | 11 +- scripts/benchmark.mjs | 20 +- site/app.js | 15 +- site/assets/runtime-map.svg | 2 +- site/benchmark.json | 28 +-- site/benchmark.md | 2 +- site/index.html | 38 ++-- src/errors.ts | 4 +- src/gitdb/database.ts | 88 ++++++++ src/gitdb/index.ts | 9 + src/gitdb/schema.ts | 165 +++++++++++++++ src/gitdb/table.ts | 89 ++++++++ src/github/github-encrypted-store.ts | 15 +- src/github/github-plaintext-store.ts | 3 + src/index.ts | 24 +-- src/orm/index.ts | 270 ------------------------ tests/benchmark-report.test.ts | 14 +- tests/benchmark-script.test.ts | 30 +-- tests/e2e.test.ts | 88 ++++---- tests/gitdb.test.ts | 175 +++++++++++++++ tests/github-tree-store.test.ts | 19 ++ tests/index-runtime.test.ts | 48 +++-- tests/no-postgres-facade.test.ts | 25 +-- tests/orm.test.ts | 192 ----------------- tests/package-metadata.test.ts | 13 +- tests/site.test.ts | 10 +- tsconfig.json | 2 - 44 files changed, 1168 insertions(+), 1031 deletions(-) delete mode 100644 examples/api-encrypted-reopen/index.mjs delete mode 100644 examples/api-local/index.mjs create mode 100644 examples/api-plaintext/index.mjs create mode 100644 src/gitdb/database.ts create mode 100644 src/gitdb/index.ts create mode 100644 src/gitdb/schema.ts create mode 100644 src/gitdb/table.ts delete mode 100644 src/orm/index.ts create mode 100644 tests/gitdb.test.ts delete mode 100644 tests/orm.test.ts diff --git a/README.md b/README.md index c7f139a..27a8bf8 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ stores sync durable state through Git tree commits. ```text App code - -> GitDB DataSource / Repository + -> GitDB database + table handles -> Local SQL engine + equality indexes -> Manifest, mutation log, page snapshots, compaction -> GitHub repository for durable history and audit @@ -26,7 +26,7 @@ Use GitDB when you want: - A database repository per project, for example `my-app-db` - Local query execution without per-query network round trips -- A TypeORM-style `DataSource` and repository API included in the package +- GitDB-native SQL and table APIs without a third-party adapter layer - Human-inspectable plaintext snapshots for public demos and reviewable data - Encrypted manifest and mutation logs for private local data - Auditable Git commits for agents, demos, content tools, config tools, and @@ -40,8 +40,8 @@ multi-node coordination. | Area | Current behavior | | --- | --- | -| App API | `createGitDbDataSource`, `defineEntity`, typed repositories | -| ORM | `save`, `insert`, `insertMany`, `saveMany`, `find`, `findOne`, `delete` | +| App API | `GitDb.open`, `openGitDb`, `defineTable`, typed table handles | +| Table API | `insert`, `insertMany`, `upsert`, `upsertMany`, `select`, `first`, `deleteWhere` | | SQL engine | `CREATE TABLE`, `INSERT`, `DELETE`, `SELECT`, joins, grouping, ordering, aggregates | | Indexes | Rebuildable primary/equality indexes for simple equality lookup | | Storage | Local plaintext, local encrypted, GitHub plaintext, GitHub encrypted | @@ -117,42 +117,47 @@ Install: npm install @3xhaust/gitdb ``` -Use the first-party repository API: +Use the first-party table API: ```ts -import { LocalPlaintextStore, createGitDbDataSource, defineEntity } from "@3xhaust/gitdb" +import { defineTable, GitDb, LocalPlaintextStore } from "@3xhaust/gitdb" +import { z } from "zod" -type Person = { - readonly id: string - readonly name: string - readonly team_id: string -} +const PersonRow = z.object({ + id: z.string(), + name: z.string(), + team_id: z.string(), +}) -const PersonEntity = defineEntity({ +const Person = defineTable({ columns: { id: "STRING", name: "STRING", team_id: "STRING" }, indexes: [{ columns: ["team_id"], name: "people_team_id_idx" }], + name: "people", primaryKey: "id", - tableName: "people", + row: PersonRow, }) -const dataSource = await createGitDbDataSource({ - entities: [PersonEntity], +const db = await GitDb.open({ store: new LocalPlaintextStore({ root: ".gitdb" }), - synchronize: true, + syncSchema: true, + tables: [Person], }) -const people = dataSource.getRepository(PersonEntity) +const people = db.table(Person) await people.insertMany([ { id: "p1", name: "Lin", team_id: "storage" }, { id: "p2", name: "Ada", team_id: "runtime" }, ]) -await people.saveMany([{ id: "p2", name: "Ada Lovelace", team_id: "runtime" }]) +await people.upsertMany([{ id: "p2", name: "Ada Lovelace", team_id: "runtime" }]) -const storagePeople = await people.find({ where: { team_id: "storage" } }) +const storagePeople = await people.select({ team_id: "storage" }) +const joined = await db.query( + "SELECT people.name FROM people WHERE people.team_id = 'storage'", +) ``` -Secondary indexes are equality-only metadata on `defineEntity`. The runtime uses -rebuilt equality indexes for simple `find({ where })` and `SELECT * ... WHERE` +Secondary indexes are equality-only metadata on `defineTable`. The runtime uses +rebuilt equality indexes for simple `select({ ... })` and `SELECT * ... WHERE` lookups when the query shape is safe; other SQL falls back to the local SQL engine. @@ -216,12 +221,13 @@ Run the bundled example: corepack pnpm example ``` -This builds the package and runs the plaintext API example plus two encrypted -API examples: +This builds the package and runs the plaintext and encrypted GitHub-backed API +examples against a separate database repository. Set `GITDB_GITHUB_OWNER`, +`GITDB_GITHUB_TOKEN`, and `GITDB_KEY` first; `GITDB_GITHUB_REPO` defaults to +`gitdb-example-db` and GitDB creates that repo when it is missing: -- `examples/api-local` +- `examples/api-plaintext` - `examples/api-encrypted` -- `examples/api-encrypted-reopen` Run local benchmarks: diff --git a/biome.json b/biome.json index a57c9ff..b658c22 100644 --- a/biome.json +++ b/biome.json @@ -47,6 +47,7 @@ "!.gitdb-example-public", "!.claude", "!.omc", + "!.omo", "!.omx" ] }, diff --git a/docs/API.md b/docs/API.md index ddfd972..6b7369f 100644 --- a/docs/API.md +++ b/docs/API.md @@ -10,38 +10,40 @@ npm install @3xhaust/gitdb GitDB is ESM-only and publishes types from `dist/src/index.d.ts`. -## Data Source +## GitDB ```ts -import { LocalPlaintextStore, createGitDbDataSource, defineEntity } from "@3xhaust/gitdb" +import { defineTable, GitDb, LocalPlaintextStore } from "@3xhaust/gitdb" +import { z } from "zod" -type Person = { - readonly id: string - readonly name: string - readonly team_id: string -} +const PersonRow = z.object({ + id: z.string(), + name: z.string(), + team_id: z.string(), +}) -const Person = defineEntity({ +const Person = defineTable({ columns: { id: "STRING", name: "STRING", team_id: "STRING" }, indexes: [{ columns: ["team_id"], name: "people_team_id_idx" }], + name: "people", primaryKey: "id", - tableName: "people", + row: PersonRow, }) -const dataSource = await createGitDbDataSource({ - entities: [Person], +const db = await GitDb.open({ store: new LocalPlaintextStore({ root: ".gitdb" }), - synchronize: true, + syncSchema: true, + tables: [Person], }) ``` -`synchronize: true` creates missing tables from entity metadata. It does not run +`syncSchema: true` creates missing tables from table metadata. It does not run destructive migrations. -## Repository +## Table Handles ```ts -const people = dataSource.getRepository(Person) +const people = db.table(Person) await people.insert({ id: "p1", name: "Lin", team_id: "storage" }) await people.insertMany([ @@ -49,28 +51,28 @@ await people.insertMany([ { id: "p3", name: "Grace", team_id: "storage" }, ]) -await people.saveMany([{ id: "p2", name: "Ada Lovelace", team_id: "runtime" }]) +await people.upsertMany([{ id: "p2", name: "Ada Lovelace", team_id: "runtime" }]) -const storagePeople = await people.find({ where: { team_id: "storage" } }) -const ada = await people.findOne({ where: { id: "p2" } }) -const deleted = await people.delete({ id: "p1" }) +const storagePeople = await people.select({ team_id: "storage" }) +const ada = await people.first({ id: "p2" }) +const deleted = await people.deleteWhere({ id: "p1" }) ``` -`insertMany()` and `saveMany()` share one transaction boundary. Prefer them over -repeated single-row `save()` calls when loading batches. +`insertMany()` and `upsertMany()` share one transaction boundary. Prefer them +over repeated single-row calls when loading batches. ## Indexes -Indexes are equality-only and rebuildable. The entity metadata records intent: +Indexes are equality-only and rebuildable. The table metadata records intent: ```ts indexes: [{ columns: ["team_id"], name: "people_team_id_idx" }] ``` The runtime maintains primary/equality indexes from committed rows. Simple -repository `find({ where })` calls and safe `SELECT * FROM table WHERE col = -literal` queries use the index path. Stale or missing index snapshot files are -ignored because indexes are derived data. +table `select({ ... })` calls and safe `SELECT * FROM table WHERE col = literal` +queries use the index path. Stale or missing index snapshot files are ignored +because indexes are derived data. ## Engine diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index cb33859..2fa2f62 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -7,10 +7,10 @@ durable history and audit. ## Runtime Layers 1. **First-party API** - - `createGitDbDataSource` - - `defineEntity` - - `GitDbRepository` - - `insert`, `insertMany`, `saveMany`, `find`, `findOne`, `delete` + - `GitDb.open` + - `openGitDb` + - `defineTable` + - `insert`, `insertMany`, `upsertMany`, `select`, `first`, `deleteWhere` 2. **SQL engine** - Local SQL execution with joins, grouping, ordering, and aggregates @@ -24,7 +24,7 @@ durable history and audit. - Additive batch commit boundary - Optional lock, snapshot, compaction, and GitHub tree-commit capabilities -4. **Repository storage** +4. **Git storage** - Local plaintext for public, inspectable page snapshots - Local encrypted for opaque manifest and log files - GitHub plaintext/encrypted for remote durable history diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 066e556..cac16b0 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -60,11 +60,11 @@ Runtime metadata: | --- | ---: | ---: | ---: | ---: | ---: | | local plaintext transaction batch | 250 | 1438.67 | 173.77 | 5.48 | 66.35 | | local encrypted transaction batch | 250 | 2173.42 | 115.03 | 0.73 | 41.65 | -| orm save() single transactions | 250 | 12304.65 | 20.32 | 0.31 | 107.08 | -| orm insert() single transactions | 250 | 8435.10 | 29.64 | 0.24 | 53.70 | -| orm insertMany bulk transaction | 250 | 1049.41 | 238.23 | 0.13 | 55.53 | -| orm saveMany bulk transaction | 250 | 2409.24 | 103.77 | 0.08 | 76.86 | -| orm indexed equality lookup | 250 | 920.12 | 271.70 | 0.08 | 45.93 | +| table upsert() single transactions | 250 | 12304.65 | 20.32 | 0.31 | 107.08 | +| table insert() single transactions | 250 | 8435.10 | 29.64 | 0.24 | 53.70 | +| table insertMany bulk transaction | 250 | 1049.41 | 238.23 | 0.13 | 55.53 | +| table upsertMany bulk transaction | 250 | 2409.24 | 103.77 | 0.08 | 76.86 | +| table indexed equality lookup | 250 | 920.12 | 271.70 | 0.08 | 45.93 | | local plaintext reopen checkpoint | 250 | 1142.73 | 218.77 | 0.74 | 36.38 | | local plaintext compaction | 250 | 105.46 | 2370.68 | 0.67 | 24.41 | @@ -77,10 +77,10 @@ The same-row previous runtime baseline is stored in | --- | ---: | ---: | ---: | ---: | ---: | | local plaintext transaction batch | 173.14 | 173.77 | +0.36% | +0.36% | +83.37% | | local encrypted transaction batch | 328.09 | 115.03 | -64.94% | -185.23% | +83.04% | -| orm indexed equality lookup | 46.49 | 271.70 | +484.44% | +82.89% | +93.88% | +| table indexed equality lookup | 46.49 | 271.70 | +484.44% | +82.89% | +93.88% | -The headline improvement is the first-party ORM path: indexed lookup plus bulk -write setup now runs at 271.70 writes/s, a 5.84x speedup over the previous ORM +The headline improvement is the first-party indexed table API path: indexed lookup plus bulk +write setup now runs at 271.70 writes/s, a 5.84x speedup over the previous indexed table API baseline. Local plaintext write throughput is roughly flat with the previous raw baseline in this run, while query latency is still much lower. Local encrypted write throughput is currently slower than the older baseline because encryption @@ -92,14 +92,14 @@ encrypted compaction remains future work. The single-row APIs remain in the benchmark because they show the cost of committing every row independently: -| Path | Writes/s | Compared with save() | +| Path | Writes/s | Compared with upsert() | | --- | ---: | ---: | -| `save()` per row | 20.32 | 1.00x | +| `upsert()` per row | 20.32 | 1.00x | | `insert()` per row | 29.64 | 1.46x | | `insertMany()` | 238.23 | 11.73x | -| `saveMany()` | 103.77 | 5.11x | +| `upsertMany()` | 103.77 | 5.11x | -For application code, prefer `insertMany()` and `saveMany()` when a workflow can +For application code, prefer `insertMany()` and `upsertMany()` when a workflow can commit a logical batch as one transaction. ## How To Read The Numbers @@ -107,7 +107,7 @@ commit a logical batch as one transaction. - `writeMs` is the time spent on the scenario's write or maintenance operation. - `writesPerSecond` normalizes `rows / writeMs`. - `Query ms` is a join for engine scenarios and an indexed equality lookup for - ORM/index scenarios. + indexed table API/index scenarios. - `reopenMs` measures a fresh runtime opening the committed store and running the validation query. - The benchmark runner emits JSON metadata for Node, platform, CPU, repeat @@ -119,10 +119,10 @@ commit a logical batch as one transaction. - local plaintext transaction batch - local encrypted transaction batch -- ORM single-row `save()` -- ORM single-row `insert()` -- ORM `insertMany()` -- ORM `saveMany()` +- indexed table API single-row `upsert()` +- indexed table API single-row `insert()` +- indexed table API `insertMany()` +- indexed table API `upsertMany()` - indexed equality lookup - reopen checkpoint - compaction diff --git a/docs/BENCHMARK_BASELINE.json b/docs/BENCHMARK_BASELINE.json index 24ab8bf..9dbdb1d 100644 --- a/docs/BENCHMARK_BASELINE.json +++ b/docs/BENCHMARK_BASELINE.json @@ -20,7 +20,7 @@ }, { "joinMs": 1.3, - "label": "orm local plaintext indexed lookup", + "label": "table indexed equality lookup", "reopenMs": 136.29, "rows": 250, "writeMs": 5377.34, diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index db2dc8c..a4c0c76 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -9,18 +9,19 @@ contract. The root package export is the stable consumer surface. Treat anything outside this list as internal unless a future release documents it here. -- `createGitDbDataSource` -- `defineEntity` -- `GitDbDataSource` -- `GitDbRepository` -- `GitDbEntityIndexDefinition` +- `GitDb` +- `openGitDb` +- `defineTable` +- `GitDbTable` +- `GitDbTableIndexDefinition` - `GitDbInsertResult` -- `GitDbSaveManyResult` +- `GitDbUpsertResult` - `GitDbEngine` - `GitDbDurabilityMode` - `GitDbEngineOptions` - `LocalPlaintextStore` - `LocalEncryptedStore` +- `GitHubPlaintextStore` - `GitHubEncryptedStore` - `GitDbStore` - `GitDbStoreV2` diff --git a/docs/README.ko.md b/docs/README.ko.md index 065ba48..8ef08c5 100644 --- a/docs/README.ko.md +++ b/docs/README.ko.md @@ -8,7 +8,7 @@ audit trail로 사용합니다. ```text Application code - -> GitDB DataSource / Repository + -> GitDB database + table handles -> Local SQL engine + transaction queue -> Manifest, mutation log, visible snapshots -> GitHub repository for durable history @@ -24,7 +24,7 @@ storage engine, transaction boundary, replay 가능한 mutation log, snapshot - 프로젝트마다 별도 database repository를 두고 싶을 때 - 매 쿼리마다 네트워크 왕복을 만들지 않고 로컬에서 실행하고 싶을 때 -- 패키지에 포함된 TypeORM 스타일 `DataSource`와 repository API가 필요할 때 +- indexed table API 계층 없이 GitDB 자체 SQL/table API로 쓰고 싶을 때 - 의도적인 public demo에서 table snapshot을 GitHub에서 바로 보고 싶을 때 - private data를 encrypted manifest와 mutation log로 저장하고 싶을 때 - agent memory, demo, content tool, config tool처럼 감사 가능한 기록이 필요한 저빈도 데이터 @@ -37,7 +37,8 @@ writer, range-heavy analytics, multi-node coordination이 필요한 워크로드 | 영역 | 현재 동작 | | --- | --- | -| App API | `createGitDbDataSource`, `defineEntity`, typed repository | +| App API | `GitDb.open`, `openGitDb`, `defineTable`, typed table handle | +| Table API | `insert`, `insertMany`, `upsert`, `upsertMany`, `select`, `first`, `deleteWhere` | | SQL engine | `CREATE TABLE`, `INSERT`, `DELETE`, `SELECT`, join, grouping, ordering, aggregate | | Storage | local encrypted, local plaintext, GitHub encrypted, GitHub plaintext | | Durability | manifest-gated mutation log replay와 visible snapshot checkpoint | @@ -45,7 +46,7 @@ writer, range-heavy analytics, multi-node coordination이 필요한 워크로드 | Compaction | local plaintext page snapshot 뒤 log segment 삭제 | | Concurrency | local single-writer, multiple readers, stale lock recovery, distributed OLTP 아님 | | CLI | `gitdb keygen`, `gitdb query`, `gitdb check` | -| Example | `examples/api-local`, `examples/api-encrypted`, `examples/api-encrypted-reopen` | +| Example | `examples/api-plaintext`, `examples/api-encrypted` | | Package | npm exports, bin, pack dry-run, publish dry-run 설정 | ## 저장소 구조 @@ -82,37 +83,39 @@ corepack pnpm install corepack pnpm build ``` -First-party repository API: +First-party table API: ```ts -import { LocalPlaintextStore, createGitDbDataSource, defineEntity } from "@3xhaust/gitdb" +import { defineTable, GitDb, LocalPlaintextStore } from "@3xhaust/gitdb" +import { z } from "zod" -type Person = { - readonly id: string - readonly name: string - readonly team_id: string -} +const PersonRow = z.object({ + id: z.string(), + name: z.string(), + team_id: z.string(), +}) -const PersonEntity = defineEntity({ +const Person = defineTable({ columns: { id: "STRING", name: "STRING", team_id: "STRING" }, indexes: [{ columns: ["team_id"], name: "people_team_id_idx" }], + name: "people", primaryKey: "id", - tableName: "people", + row: PersonRow, }) -const dataSource = await createGitDbDataSource({ - entities: [PersonEntity], +const db = await GitDb.open({ store: new LocalPlaintextStore({ root: ".gitdb" }), - synchronize: true, + syncSchema: true, + tables: [Person], }) -const people = dataSource.getRepository(PersonEntity) -await people.save({ id: "p1", name: "Lin", team_id: "storage" }) -const storagePeople = await people.find({ where: { team_id: "storage" } }) +const people = db.table(Person) +await people.upsert({ id: "p1", name: "Lin", team_id: "storage" }) +const storagePeople = await people.select({ team_id: "storage" }) ``` -Secondary index는 `defineEntity`의 equality-only metadata로 선언합니다. 런타임은 -committed row에서 equality index를 재빌드하고, 안전한 `find({ where })`와 단순 +Secondary index는 `defineTable`의 equality-only metadata로 선언합니다. 런타임은 +committed row에서 equality index를 재빌드하고, 안전한 `select({ ... })`와 단순 `SELECT * ... WHERE` 조회에 사용합니다. `indexes.json`이 stale이어도 query result는 committed row 기준으로 재빌드됩니다. @@ -122,9 +125,12 @@ result는 committed row 기준으로 재빌드됩니다. corepack pnpm example ``` -예제는 패키지를 빌드한 뒤 plaintext API 예제와 암호화 API 예제 2개를 차례로 -실행합니다. 각 예제는 JSON summary를 출력하며, 암호화 reopen 예제는 잘못된 키 -거부와 plaintext 누출 없음까지 확인합니다. +예제는 패키지를 빌드한 뒤 별도 database repo에 GitHub-backed plaintext API 예제와 +암호화 API 예제를 차례로 실행합니다. `GITDB_GITHUB_OWNER`, +`GITDB_GITHUB_TOKEN`, `GITDB_KEY`를 먼저 설정해야 하며, +`GITDB_GITHUB_REPO`를 생략하면 `gitdb-example-db` repo를 사용하거나 새로 +만듭니다. 각 예제는 원격 manifest 파일 존재와 GitDB read-back을 확인한 JSON +summary를 출력합니다. ## CLI @@ -182,8 +188,8 @@ table 이름, column, row가 공개되어도 되는 demo에서만 사용하세 GitDB는 네 계층으로 나뉩니다: 1. First-party API - - `DataSource`, `Repository`, `save`, `find`, `findOne`, `delete`, raw - `query`, transaction access를 제공합니다. + - `GitDb.open`, `defineTable`, table handle, raw `query`, transaction + access를 제공합니다. - 새 앱이 GitDB runtime 표면에 직접 붙도록 합니다. 2. SQL engine @@ -228,11 +234,11 @@ GITDB_BENCH_ROWS=250 corepack pnpm benchmark:compare | --- | ---: | ---: | ---: | ---: | ---: | | local plaintext throttled visible snapshots | 25 | 646.19 | 38.69 | 10.26 | 20.38 | | local encrypted mutation log | 25 | 650.07 | 38.46 | 2.06 | 24.26 | -| orm local plaintext indexed lookup | 25 | 1012.42 | 24.69 | 1.47 | 32.68 | +| table indexed equality lookup | 25 | 1012.42 | 24.69 | 1.47 | 32.68 | | local plaintext compaction | 25 | 96.23 | 259.80 | 2.58 | 11.56 | 해석: raw local execution은 demo와 저빈도 프로젝트 데이터에 충분히 빠릅니다. -Repository bulk write는 `insertMany()`나 `saveMany()`를 쓰고, query 측정에는 +Table bulk write는 `insertMany()`나 `upsertMany()`를 쓰고, query 측정에는 equality index lookup이 포함됩니다. Compaction은 checkpoint 뒤 old log를 삭제해 reopen path를 짧게 만듭니다. @@ -241,8 +247,8 @@ reopen path를 짧게 만듭니다. ## 현재 한계 - SQL 지원 범위는 현재 실행 가능한 subset으로 제한됩니다. -- Single-row `save()`는 bulk path가 아닙니다. Repository batch write는 - `insertMany()`나 `saveMany()`를 사용합니다. +- Single-row `upsert()`는 bulk path가 아닙니다. Table batch write는 + `insertMany()`나 `upsertMany()`를 사용합니다. - Local multi-process write는 single-writer, multiple readers, stale lock recovery를 지원합니다. Distributed OLTP는 아닙니다. - GitHub tree commit은 durable audit sync용이지 SELECT hot path나 distributed OLTP가 아닙니다. diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 154a6d2..125a665 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -14,7 +14,7 @@ Summary: runtime over repository-backed durability. - Adds store contract v2, manifest-gated batch commits, local writer locks, GitHub tree-commit sync, page snapshots, equality indexes, compaction, and a - first-party repository API with bulk write methods. + first-party table API with bulk write methods. - Publishes benchmark evidence, a rebuilt website, npm dry-run scripts, and a trusted publishing workflow that keeps live npm publish credential-gated. @@ -38,7 +38,7 @@ shell over files. The runtime owns: - rebuildable primary and secondary equality indexes; - local plaintext compaction with checkpoint-first ordering; - GitHub Git Database tree, commit, and non-force ref update sync; -- typed `DataSource` and repository APIs for app code. +- typed `GitDb` and table APIs for app code. The public contract is documented in `docs/ENGINE_CONTRACT.md`, `docs/ARCHITECTURE.md`, `docs/API.md`, and `docs/MIGRATION.md`. @@ -52,17 +52,17 @@ Benchmark evidence was refreshed on 2026-06-15 KST with | --- | ---: | ---: | ---: | | local plaintext transaction batch | 173.77 | 5.48 | 66.35 | | local encrypted transaction batch | 115.03 | 0.73 | 41.65 | -| `save()` single transactions | 20.32 | 0.31 | 107.08 | +| `upsert()` single transactions | 20.32 | 0.31 | 107.08 | | `insertMany()` bulk transaction | 238.23 | 0.13 | 55.53 | | indexed equality lookup | 271.70 | 0.08 | 45.93 | | local plaintext compaction | 2370.68 | 0.67 | 24.41 | Compared with the previous same-row baseline: -- indexed ORM lookup improved from 46.49 writes/s to 271.70 writes/s +- indexed table lookup improved from 46.49 writes/s to 271.70 writes/s (`+484.44%`, 5.84x); -- `insertMany()` is 11.73x faster than per-row `save()`; -- `saveMany()` is 5.11x faster than per-row `save()`; +- `insertMany()` is 11.73x faster than per-row `upsert()`; +- `upsertMany()` is 5.11x faster than per-row `upsert()`; - local plaintext write throughput is roughly flat in this run, but query latency improved sharply; - encrypted write throughput is slower while encrypted compaction remains a @@ -75,10 +75,10 @@ Full data lives in `docs/BENCHMARKS.md`, `site/benchmark.json`, and Existing users should migrate toward the first-party runtime surface: -- use `createGitDbDataSource`, `defineEntity`, and repositories from the root +- use `GitDb.open`, `openGitDb`, `defineTable`, and table handles from the root package export; -- prefer `insertMany()` or `saveMany()` for logical batches; -- declare equality indexes in entity metadata when using frequent exact-match +- prefer `insertMany()` or `upsertMany()` for logical batches; +- declare equality indexes in table metadata when using frequent exact-match lookups; - treat visible snapshots, page files, and index files as derived data; - use `docs/MIGRATION.md` for manifest v1 to store contract v2 behavior. diff --git a/examples/api-encrypted-reopen/index.mjs b/examples/api-encrypted-reopen/index.mjs deleted file mode 100644 index c52c807..0000000 --- a/examples/api-encrypted-reopen/index.mjs +++ /dev/null @@ -1,95 +0,0 @@ -import { mkdtemp, readdir, readFile, rm } from "node:fs/promises" -import { tmpdir } from "node:os" -import { join } from "node:path" -import { - createAesGcmCipher, - createGitDbDataSource, - defineEntity, - LocalEncryptedStore, -} from "../../dist/src/index.js" - -const Secret = defineEntity({ - columns: { - id: "STRING", - payload: "STRING", - }, - primaryKey: "id", - tableName: "secrets", -}) - -const root = await mkdtemp(join(tmpdir(), "gitdb-api-encrypted-reopen-")) -const key = Buffer.alloc(32, 11).toString("base64url") -const wrongKey = Buffer.alloc(32, 12).toString("base64url") -const secretValue = "launch-code-4271" - -try { - const entities = [Secret] - const dataSource = await createGitDbDataSource({ - entities, - store: new LocalEncryptedStore({ cipher: createAesGcmCipher(key), root }), - synchronize: true, - }) - - await dataSource.getRepository(Secret).insert({ id: "secret_1", payload: secretValue }) - - const reopened = await createGitDbDataSource({ - entities, - store: new LocalEncryptedStore({ cipher: createAesGcmCipher(key), root }), - synchronize: false, - }) - const rows = await reopened.getRepository(Secret).find() - const storedText = await readTree(root) - const leaksPlaintext = storedText.includes(secretValue) - const wrongKeyRejected = await rejectsWrongKey(entities) - - if (leaksPlaintext) { - throw new Error("encrypted example leaked plaintext into stored files") - } - if (!wrongKeyRejected) { - throw new Error("encrypted example did not reject a wrong key") - } - - const summary = { - example: "api-encrypted-reopen", - leaksPlaintext, - mode: "local-encrypted", - reopenedRows: rows, - status: "ok", - wrongKeyRejected, - } - process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`) -} finally { - await rm(root, { force: true, recursive: true }) -} - -async function rejectsWrongKey(entities) { - try { - await createGitDbDataSource({ - entities, - store: new LocalEncryptedStore({ cipher: createAesGcmCipher(wrongKey), root }), - synchronize: false, - }) - return false - } catch (error) { - if (error instanceof Error) { - return true - } - throw error - } -} - -async function readTree(path) { - const entries = await readdir(path, { withFileTypes: true }) - const chunks = [] - for (const entry of entries) { - const child = join(path, entry.name) - if (entry.isDirectory()) { - chunks.push(await readTree(child)) - continue - } - if (entry.isFile()) { - chunks.push(await readFile(child, "utf8")) - } - } - return chunks.join("\n") -} diff --git a/examples/api-encrypted/index.mjs b/examples/api-encrypted/index.mjs index 789882e..39f3f57 100644 --- a/examples/api-encrypted/index.mjs +++ b/examples/api-encrypted/index.mjs @@ -1,55 +1,117 @@ -import { mkdtemp, rm } from "node:fs/promises" -import { tmpdir } from "node:os" -import { join } from "node:path" +import { Octokit } from "@octokit/rest" +import { z } from "zod" import { createAesGcmCipher, - createGitDbDataSource, - defineEntity, - LocalEncryptedStore, + defineTable, + GitDb, + GitHubEncryptedStore, } from "../../dist/src/index.js" -const Account = defineEntity({ +const DEFAULT_DATABASE_REPO = "gitdb-example-db" + +const AccountRow = z.object({ + id: z.string(), + status: z.string(), + tier: z.string(), +}) + +const Account = defineTable({ columns: { id: "STRING", - tier: "STRING", status: "STRING", + tier: "STRING", }, indexes: [{ columns: ["status"], name: "accounts_status_idx" }], + name: "accounts", primaryKey: "id", - tableName: "accounts", + row: AccountRow, }) -const key = Buffer.alloc(32, 7).toString("base64url") -const root = await mkdtemp(join(tmpdir(), "gitdb-api-encrypted-")) +const config = githubConfig("encrypted") +const cipher = createAesGcmCipher(requiredEnv("GITDB_KEY")) +const store = new GitHubEncryptedStore(config, cipher) +const db = await GitDb.open({ store, syncSchema: true, tables: [Account] }) +const accounts = db.table(Account) + +await accounts.upsertMany([ + { id: "acct_1", status: "active", tier: "team" }, + { id: "acct_2", status: "active", tier: "pro" }, +]) -try { - const entities = [Account] - const dataSource = await createGitDbDataSource({ - entities, - store: new LocalEncryptedStore({ cipher: createAesGcmCipher(key), root }), - synchronize: true, +const remoteManifestPath = `${config.prefix}/manifest.enc` +const remotePayload = await readGitHubFile(config, remoteManifestPath) +const leaksPlaintext = remotePayload.includes("acct_1") || remotePayload.includes("active") +if (leaksPlaintext) { + throw new Error("encrypted GitHub example leaked plaintext into the remote manifest") +} + +const activeAccounts = await readRowsFromGitHub(async () => { + const verificationDb = await GitDb.open({ + store: new GitHubEncryptedStore(config, cipher), + tables: [Account], }) - const accounts = dataSource.getRepository(Account) - - await accounts.insertMany([ - { id: "acct_1", tier: "team", status: "active" }, - { id: "acct_2", tier: "free", status: "trial" }, - ]) - await accounts.saveMany([{ id: "acct_2", tier: "pro", status: "active" }]) - - const reopened = await createGitDbDataSource({ - entities, - store: new LocalEncryptedStore({ cipher: createAesGcmCipher(key), root }), - synchronize: false, + return await verificationDb.table(Account).select({ status: "active" }) +}, 2) + +const summary = { + activeAccounts, + branch: config.branch, + example: "api-encrypted", + leaksPlaintext, + mode: "github-encrypted", + prefix: config.prefix, + remoteManifestPath, + remoteVerified: true, + repo: `${config.owner}/${config.repo}`, + status: "ok", +} + +process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`) + +function githubConfig(mode) { + const basePrefix = requiredEnv("GITDB_GITHUB_PREFIX", "gitdb/v1") + return { + branch: requiredEnv("GITDB_GITHUB_BRANCH", "main"), + owner: requiredEnv("GITDB_GITHUB_OWNER"), + prefix: `${basePrefix}/examples/${mode}`, + repo: requiredEnv("GITDB_GITHUB_REPO", DEFAULT_DATABASE_REPO), + token: requiredEnv("GITDB_GITHUB_TOKEN"), + } +} + +function requiredEnv(name, fallback) { + const value = process.env[name] ?? fallback + if (value === undefined || value.trim().length === 0) { + throw new Error(`${name} is required for GitHub-backed examples`) + } + return value +} + +async function readGitHubFile(config, path) { + const octokit = new Octokit({ auth: config.token }) + const response = await octokit.repos.getContent({ + owner: config.owner, + path, + ref: config.branch, + repo: config.repo, }) + if (Array.isArray(response.data) || response.data.type !== "file") { + throw new Error(`GitHub path is not a file: ${path}`) + } + return Buffer.from(response.data.content, "base64").toString("utf8") +} - const summary = { - activeAccounts: await reopened.getRepository(Account).find({ where: { status: "active" } }), - example: "api-encrypted", - mode: "local-encrypted", - status: "ok", +async function readRowsFromGitHub(selectRows, expectedCount) { + for (let attempt = 0; attempt < 8; attempt += 1) { + const rows = await selectRows() + if (rows.length === expectedCount) { + return rows + } + await delay(750) } - process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`) -} finally { - await rm(root, { force: true, recursive: true }) + throw new Error(`GitHub read-back did not return ${expectedCount} rows`) +} + +async function delay(milliseconds) { + await new Promise((resolve) => setTimeout(resolve, milliseconds)) } diff --git a/examples/api-local/index.mjs b/examples/api-local/index.mjs deleted file mode 100644 index 50fc9f5..0000000 --- a/examples/api-local/index.mjs +++ /dev/null @@ -1,69 +0,0 @@ -import { mkdtemp, rm } from "node:fs/promises" -import { tmpdir } from "node:os" -import { join } from "node:path" -import { createGitDbDataSource, defineEntity, LocalPlaintextStore } from "../../dist/src/index.js" - -const Team = defineEntity({ - columns: { - id: "STRING", - name: "STRING", - }, - primaryKey: "id", - tableName: "teams", -}) - -const Person = defineEntity({ - columns: { - id: "STRING", - name: "STRING", - team_id: "STRING", - }, - indexes: [{ columns: ["team_id"], name: "people_team_id_idx" }], - primaryKey: "id", - tableName: "people", -}) - -const root = await mkdtemp(join(tmpdir(), "gitdb-api-local-")) - -try { - const entities = [Team, Person] - const dataSource = await createGitDbDataSource({ - entities, - store: new LocalPlaintextStore({ root }), - synchronize: true, - }) - - const teams = dataSource.getRepository(Team) - const people = dataSource.getRepository(Person) - - await teams.insertMany([ - { id: "t1", name: "Storage" }, - { id: "t2", name: "Runtime" }, - ]) - await people.insertMany([ - { id: "p1", name: "Lin", team_id: "t1" }, - { id: "p2", name: "Ada", team_id: "t2" }, - { id: "p3", name: "Grace", team_id: "t1" }, - ]) - await people.saveMany([{ id: "p2", name: "Ada Lovelace", team_id: "t2" }]) - - const joined = await dataSource.query( - "SELECT people.name AS person, teams.name AS team FROM people JOIN teams ON people.team_id = teams.id", - ) - const reopened = await createGitDbDataSource({ - entities, - store: new LocalPlaintextStore({ root }), - synchronize: false, - }) - - const summary = { - example: "api-local", - joined, - mode: "local-plaintext", - reopenedPeople: await reopened.getRepository(Person).find({ where: { team_id: "t1" } }), - status: "ok", - } - process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`) -} finally { - await rm(root, { force: true, recursive: true }) -} diff --git a/examples/api-plaintext/index.mjs b/examples/api-plaintext/index.mjs new file mode 100644 index 0000000..fb1a90d --- /dev/null +++ b/examples/api-plaintext/index.mjs @@ -0,0 +1,105 @@ +import { Octokit } from "@octokit/rest" +import { z } from "zod" +import { defineTable, GitDb, GitHubPlaintextStore } from "../../dist/src/index.js" + +const DEFAULT_DATABASE_REPO = "gitdb-example-db" + +const PersonRow = z.object({ + id: z.string(), + name: z.string(), + team: z.string(), +}) + +const Person = defineTable({ + columns: { + id: "STRING", + name: "STRING", + team: "STRING", + }, + indexes: [{ columns: ["team"], name: "people_team_idx" }], + name: "people", + primaryKey: "id", + row: PersonRow, +}) + +const config = githubConfig("plaintext") +const store = new GitHubPlaintextStore(config) +const db = await GitDb.open({ store, syncSchema: true, tables: [Person] }) +const people = db.table(Person) + +await people.upsertMany([ + { id: "p1", name: "Lin", team: "storage" }, + { id: "p2", name: "Ada", team: "runtime" }, + { id: "p3", name: "Grace", team: "storage" }, +]) + +const remoteManifestPath = `${config.prefix}/manifest.json` +await readGitHubFile(config, remoteManifestPath) +const rows = await readRowsFromGitHub(async () => { + const verificationDb = await GitDb.open({ + store: new GitHubPlaintextStore(config), + tables: [Person], + }) + return await verificationDb.table(Person).select({ team: "storage" }) +}, 2) + +const summary = { + branch: config.branch, + example: "api-plaintext", + mode: "github-plaintext", + prefix: config.prefix, + remoteManifestPath, + remoteVerified: true, + repo: `${config.owner}/${config.repo}`, + rows, + status: "ok", +} +process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`) + +function githubConfig(mode) { + const basePrefix = requiredEnv("GITDB_GITHUB_PREFIX", "gitdb/v1") + return { + branch: requiredEnv("GITDB_GITHUB_BRANCH", "main"), + owner: requiredEnv("GITDB_GITHUB_OWNER"), + prefix: `${basePrefix}/examples/${mode}`, + repo: requiredEnv("GITDB_GITHUB_REPO", DEFAULT_DATABASE_REPO), + token: requiredEnv("GITDB_GITHUB_TOKEN"), + } +} + +function requiredEnv(name, fallback) { + const value = process.env[name] ?? fallback + if (value === undefined || value.trim().length === 0) { + throw new Error(`${name} is required for GitHub-backed examples`) + } + return value +} + +async function readGitHubFile(config, path) { + const octokit = new Octokit({ auth: config.token }) + const response = await octokit.repos.getContent({ + owner: config.owner, + path, + ref: config.branch, + repo: config.repo, + }) + if (Array.isArray(response.data) || response.data.type !== "file") { + throw new Error(`GitHub path is not a file: ${path}`) + } + return Buffer.from(response.data.content, "base64").toString("utf8") +} + +async function readRowsFromGitHub(selectRows, expectedCount) { + for (let attempt = 0; attempt < 8; attempt += 1) { + const rows = await selectRows() + if (rows.length === expectedCount) { + return rows + } + await delay(750) + } + throw new Error(`GitHub read-back did not return ${expectedCount} rows`) +} + +async function delay(milliseconds) { + await new Promise((resolve) => setTimeout(resolve, milliseconds)) +} diff --git a/package.json b/package.json index e7c4b0e..3660498 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@3xhaust/gitdb", "version": "0.1.0", - "description": "GitHub-backed local database runtime with auditable storage, transactions, and ORM APIs.", + "description": "Git-backed local database runtime with auditable storage, SQL, and table APIs.", "type": "module", "main": "./dist/src/index.js", "types": "./dist/src/index.d.ts", @@ -31,10 +31,9 @@ "benchmark:github": "corepack pnpm build && node scripts/benchmark.mjs --github", "check": "biome check . && tsc --noEmit", "example": "corepack pnpm example:api", - "example:api": "corepack pnpm build && node examples/api-local/index.mjs && node examples/api-encrypted/index.mjs && node examples/api-encrypted-reopen/index.mjs", - "example:api-local": "corepack pnpm build && node examples/api-local/index.mjs", + "example:api": "corepack pnpm build && node examples/api-plaintext/index.mjs && node examples/api-encrypted/index.mjs", + "example:api-plaintext": "corepack pnpm build && node examples/api-plaintext/index.mjs", "example:api-encrypted": "corepack pnpm build && node examples/api-encrypted/index.mjs", - "example:api-encrypted-reopen": "corepack pnpm build && node examples/api-encrypted-reopen/index.mjs", "format": "biome check --write .", "pack:dry-run": "corepack pnpm build && COREPACK_ENABLE_STRICT=0 npm pack --dry-run --json", "publish:dry-run": "corepack pnpm build && COREPACK_ENABLE_STRICT=0 npm publish --dry-run --access public", diff --git a/scripts/benchmark-gate.mjs b/scripts/benchmark-gate.mjs index d198d9d..db88b9a 100644 --- a/scripts/benchmark-gate.mjs +++ b/scripts/benchmark-gate.mjs @@ -7,11 +7,11 @@ const execFileAsync = promisify(execFile) const CURRENT_REQUIRED_SCENARIOS = [ "local-plaintext", "local-encrypted", - "orm-save-single", - "orm-insert", - "orm-insert-many", - "orm-save-many", - "orm-indexed-lookup", + "table-upsert-single", + "table-insert", + "table-insert-many", + "table-upsert-many", + "table-indexed-lookup", "reopen", "compaction", ] @@ -181,12 +181,13 @@ function positiveFiniteNumber(value, field) { function scenarioKey(label) { const normalized = label.toLowerCase() - if (normalized.includes("save()")) return "orm-save-single" + if (normalized.includes("upsert()")) return "table-upsert-single" if (normalized.includes("insertmany") || normalized.includes("insert many")) - return "orm-insert-many" - if (normalized.includes("savemany") || normalized.includes("save many")) return "orm-save-many" - if (normalized.includes("insert()")) return "orm-insert" - if (normalized.includes("indexed")) return "orm-indexed-lookup" + return "table-insert-many" + if (normalized.includes("upsertmany") || normalized.includes("upsert many")) + return "table-upsert-many" + if (normalized.includes("insert()")) return "table-insert" + if (normalized.includes("indexed")) return "table-indexed-lookup" if (normalized.includes("compaction")) return "compaction" if (normalized.includes("reopen checkpoint")) return "reopen" if (normalized.startsWith("local plaintext")) return "local-plaintext" diff --git a/scripts/benchmark-report.mjs b/scripts/benchmark-report.mjs index 058a358..66c24e9 100644 --- a/scripts/benchmark-report.mjs +++ b/scripts/benchmark-report.mjs @@ -161,23 +161,20 @@ function compareRows(key, baselineRow, currentRow) { function scenarioKey(label) { const normalized = label.toLowerCase() - if (normalized.includes("save()")) { - return "orm save single" + if (normalized.includes("upsert()")) { + return "table upsert single" } if (normalized.includes("insertmany") || normalized.includes("insert many")) { - return "orm insert many" + return "table insert many" } - if (normalized.includes("savemany") || normalized.includes("save many")) { - return "orm save many" + if (normalized.includes("upsertmany") || normalized.includes("upsert many")) { + return "table upsert many" } if (normalized.includes("insert()")) { - return "orm insert" + return "table insert" } if (normalized.includes("indexed")) { - return "orm indexed lookup" - } - if (normalized.startsWith("orm local plaintext")) { - return "orm indexed lookup" + return "table indexed lookup" } if (normalized.includes("reopen checkpoint")) { return "reopen" @@ -202,11 +199,11 @@ function siteScenarioKey(label) { } function headline(comparisons) { - const orm = comparisons.find((row) => row.key === "orm save single") + const table = comparisons.find((row) => row.key === "table upsert single") const raw = comparisons.find((row) => row.key === "local plaintext") - if (orm !== undefined && raw !== undefined) { - const overhead = ratio(raw.current.writesPerSecond, orm.current.writesPerSecond) - return `Raw engine is ${formatRatio(overhead)} of ORM writes/s (save() transaction overhead)` + if (table !== undefined && raw !== undefined) { + const overhead = ratio(raw.current.writesPerSecond, table.current.writesPerSecond) + return `Raw SQL is ${formatRatio(overhead)} of table upsert writes/s` } const localPlaintext = comparisons.find((row) => row.key === "local plaintext") if (localPlaintext === undefined) { diff --git a/scripts/benchmark-scenarios.mjs b/scripts/benchmark-scenarios.mjs index 70dae2f..2420522 100644 --- a/scripts/benchmark-scenarios.mjs +++ b/scripts/benchmark-scenarios.mjs @@ -1,25 +1,39 @@ import { mkdtemp, rm } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" +import { z } from "zod" import { createAesGcmCipher } from "../dist/src/crypto/aes-gcm.js" import { GitHubPlaintextStore } from "../dist/src/github/github-plaintext-store.js" -import { createGitDbDataSource, defineEntity } from "../dist/src/orm/index.js" +import { defineTable, GitDb } from "../dist/src/index.js" import { GitDbEngine } from "../dist/src/sql/engine.js" import { LocalEncryptedStore } from "../dist/src/storage/local-encrypted-store.js" import { LocalPlaintextStore } from "../dist/src/storage/local-plaintext-store.js" import { requiredEnv, result, time } from "./benchmark-utils.mjs" -const Team = defineEntity({ +const TeamRow = z.object({ + id: z.string(), + name: z.string(), +}) + +const PersonRow = z.object({ + id: z.string(), + name: z.string(), + team_id: z.string(), +}) + +const Team = defineTable({ columns: { id: "STRING", name: "STRING" }, + name: "teams", primaryKey: "id", - tableName: "teams", + row: TeamRow, }) -const Person = defineEntity({ +const Person = defineTable({ columns: { id: "STRING", name: "STRING", team_id: "STRING" }, indexes: [{ columns: ["team_id"], name: "people_team_id_idx" }], + name: "people", primaryKey: "id", - tableName: "people", + row: PersonRow, }) export async function benchEngine({ encrypted = false, rows, snapshotPolicy }) { @@ -43,21 +57,21 @@ export async function benchEngine({ encrypted = false, rows, snapshotPolicy }) { }) } -export async function benchOrmSaveSingle(options) { - return await benchOrmWrite({ - label: "orm save() single transactions", +export async function benchTableUpsertSingle(options) { + return await benchTableWrite({ + label: "table upsert() single transactions", options, write: async ({ people, rows }) => { for (const row of peopleRows(rows)) { - await people.save(row) + await people.upsert(row) } }, }) } -export async function benchOrmInsertSingle(options) { - return await benchOrmWrite({ - label: "orm insert() single transactions", +export async function benchTableInsertSingle(options) { + return await benchTableWrite({ + label: "table insert() single transactions", options, write: async ({ people, rows }) => { for (const row of peopleRows(rows)) { @@ -67,9 +81,9 @@ export async function benchOrmInsertSingle(options) { }) } -export async function benchOrmInsertMany(options) { - return await benchOrmWrite({ - label: "orm insertMany bulk transaction", +export async function benchTableInsertMany(options) { + return await benchTableWrite({ + label: "table insertMany bulk transaction", options, write: async ({ people, rows }) => { await people.insertMany(peopleRows(rows)) @@ -77,19 +91,19 @@ export async function benchOrmInsertMany(options) { }) } -export async function benchOrmSaveMany(options) { - return await benchOrmWrite({ - label: "orm saveMany bulk transaction", +export async function benchTableUpsertMany(options) { + return await benchTableWrite({ + label: "table upsertMany bulk transaction", options, write: async ({ people, rows }) => { - await people.saveMany(peopleRows(rows)) + await people.upsertMany(peopleRows(rows)) }, }) } -export async function benchOrmIndexedLookup(options) { - return await benchOrmWrite({ - label: "orm indexed equality lookup", +export async function benchTableIndexedLookup(options) { + return await benchTableWrite({ + label: "table indexed equality lookup", options, write: async ({ people, rows }) => { await people.insertMany(peopleRows(rows)) @@ -152,31 +166,31 @@ export async function benchGitHubPlaintext(rowCount) { return result(`github plaintext tree commits (${prefix})`, rowCount, writeMs, queryMs, reopenMs) } -async function benchOrmWrite({ label, options, write }) { - return await withRoot("gitdb-orm-bench-", async (root) => { - const dataSource = await ormDataSource(root, options.snapshotPolicy, true) - const people = dataSource.getRepository(Person) +async function benchTableWrite({ label, options, write }) { + return await withRoot("gitdb-table-bench-", async (root) => { + const db = await tableDb(root, options.snapshotPolicy, true) + const people = db.table(Person) const writeMs = await time(async () => write({ people, rows: options.rows })) const queryMs = await time(async () => assertIndexedLookup(people)) const reopenMs = await time(async () => { - const reopened = await ormDataSource(root, options.snapshotPolicy, false) - await assertOrmJoin(reopened, options.rows) + const reopened = await tableDb(root, options.snapshotPolicy, false) + await assertTableJoin(reopened, options.rows) }) return result(label, options.rows, writeMs, queryMs, reopenMs) }) } -async function ormDataSource(root, snapshotPolicy, synchronize) { - const dataSource = await createGitDbDataSource({ - entities: [Team, Person], +async function tableDb(root, snapshotPolicy, syncSchema) { + const db = await GitDb.open({ snapshotPolicy, store: new LocalPlaintextStore({ root }), - synchronize, + syncSchema, + tables: [Team, Person], }) - if (synchronize) { - await seedTeams(dataSource.getRepository(Team)) + if (syncSchema) { + await seedTeams(db.table(Team)) } - return dataSource + return db } async function seedTeams(teams) { @@ -217,14 +231,14 @@ function peopleRows(rowCount) { } async function assertIndexedLookup(people) { - const rows = await people.find({ where: { team_id: "t1" } }) + const rows = await people.select({ team_id: "t1" }) if (rows.length === 0 || rows.some((row) => row.team_id !== "t1")) { throw new Error("indexed lookup returned unexpected rows") } } -async function assertOrmJoin(dataSource, rowCount) { - const rows = await dataSource.query( +async function assertTableJoin(db, rowCount) { + const rows = await db.query( "SELECT people.name AS person, teams.name AS team FROM people JOIN teams ON people.team_id = teams.id ORDER BY people.name", ) if (rows.length !== rowCount) { diff --git a/scripts/benchmark-utils.mjs b/scripts/benchmark-utils.mjs index 62dc0d6..9eb0139 100644 --- a/scripts/benchmark-utils.mjs +++ b/scripts/benchmark-utils.mjs @@ -21,12 +21,13 @@ export function result(label, rowCount, writeMs, joinMs, reopenMs) { export function scenarioKey(label) { const normalized = label.toLowerCase() - if (normalized.includes("savemany") || normalized.includes("save many")) return "orm-save-many" - if (normalized.includes("save()")) return "orm-save-single" + if (normalized.includes("upsertmany") || normalized.includes("upsert many")) + return "table-upsert-many" + if (normalized.includes("upsert()")) return "table-upsert-single" if (normalized.includes("insertmany") || normalized.includes("insert many")) - return "orm-insert-many" - if (normalized.includes("insert()")) return "orm-insert" - if (normalized.includes("indexed")) return "orm-indexed-lookup" + return "table-insert-many" + if (normalized.includes("insert()")) return "table-insert" + if (normalized.includes("indexed")) return "table-indexed-lookup" if (normalized.includes("compaction")) return "compaction" if (normalized.includes("reopen checkpoint")) return "reopen" if (normalized.startsWith("local plaintext")) return "local-plaintext" diff --git a/scripts/benchmark.mjs b/scripts/benchmark.mjs index a4e96b0..1a58976 100644 --- a/scripts/benchmark.mjs +++ b/scripts/benchmark.mjs @@ -3,12 +3,12 @@ import { benchCompaction, benchEngine, benchGitHubPlaintext, - benchOrmIndexedLookup, - benchOrmInsertMany, - benchOrmInsertSingle, - benchOrmSaveMany, - benchOrmSaveSingle, benchReopen, + benchTableIndexedLookup, + benchTableInsertMany, + benchTableInsertSingle, + benchTableUpsertMany, + benchTableUpsertSingle, } from "./benchmark-scenarios.mjs" import { formatMarkdown, numberEnv, result, scenarioKey, writeJson } from "./benchmark-utils.mjs" @@ -22,11 +22,11 @@ const snapshotPolicy = { mode: "interval", mutations: 100 } const localScenarios = [ () => benchEngine({ rows, snapshotPolicy }), () => benchEngine({ encrypted: true, rows }), - () => benchOrmSaveSingle({ rows, snapshotPolicy }), - () => benchOrmInsertSingle({ rows, snapshotPolicy }), - () => benchOrmInsertMany({ rows, snapshotPolicy }), - () => benchOrmSaveMany({ rows, snapshotPolicy }), - () => benchOrmIndexedLookup({ rows, snapshotPolicy }), + () => benchTableUpsertSingle({ rows, snapshotPolicy }), + () => benchTableInsertSingle({ rows, snapshotPolicy }), + () => benchTableInsertMany({ rows, snapshotPolicy }), + () => benchTableUpsertMany({ rows, snapshotPolicy }), + () => benchTableIndexedLookup({ rows, snapshotPolicy }), () => benchReopen({ rows, snapshotPolicy }), () => benchCompaction({ rows, snapshotPolicy }), ] diff --git a/site/app.js b/site/app.js index 3294a55..7e67572 100644 --- a/site/app.js +++ b/site/app.js @@ -27,11 +27,14 @@ function render(data) { const scenarios = data.scenarios ?? [] const comparisons = data.comparisons ?? [] const local = findScenario(scenarios, "local-plaintext") - const saveSingle = findScenario(scenarios, "orm-save-single") - const insertMany = findScenario(scenarios, "orm-insert-many") - const indexed = findScenario(scenarios, "orm-indexed-lookup") + const upsertSingle = findScenario(scenarios, "table-upsert-single") + const insertMany = findScenario(scenarios, "table-insert-many") + const indexed = findScenario(scenarios, "table-indexed-lookup") setText("hero-raw-wps", formatNumber(local?.writesPerSecond)) - setText("hero-bulk-delta", formatRatio(insertMany?.writesPerSecond, saveSingle?.writesPerSecond)) + setText( + "hero-bulk-delta", + formatRatio(insertMany?.writesPerSecond, upsertSingle?.writesPerSecond), + ) setText( "hero-index-ms", indexed?.joinMs === undefined ? "n/a" : `${formatNumber(indexed.joinMs)} ms`, @@ -130,10 +133,10 @@ function bar(key, value, max) { } function barTone(key) { - if (key === "orm-insert-many" || key === "orm-indexed-lookup") { + if (key === "table-insert-many" || key === "table-indexed-lookup") { return "accent" } - if (key === "orm-save-single" || key === "orm-insert") { + if (key === "table-upsert-single" || key === "table-insert") { return "slow" } return "current" diff --git a/site/assets/runtime-map.svg b/site/assets/runtime-map.svg index a10ca30..abc4bdf 100644 --- a/site/assets/runtime-map.svg +++ b/site/assets/runtime-map.svg @@ -9,7 +9,7 @@ App code - ORM repository + GitDB table SQL engine API diff --git a/site/benchmark.json b/site/benchmark.json index 3478208..93cfd8a 100644 --- a/site/benchmark.json +++ b/site/benchmark.json @@ -59,7 +59,7 @@ { "baseline": { "joinMs": 1.3, - "label": "orm local plaintext indexed lookup", + "label": "table indexed equality lookup", "reopenMs": 136.29, "rows": 250, "writeMs": 5377.34, @@ -67,17 +67,17 @@ }, "current": { "joinMs": 0.060459000000264496, - "label": "orm indexed equality lookup", + "label": "table indexed equality lookup", "reopenMs": 45.7231249999968, "rows": 250, "writeMs": 1015.2725830000018, "writesPerSecond": 246.23929000552906 }, "joinMsChangePct": 95.34930769228734, - "key": "orm indexed lookup", + "key": "table indexed lookup", "reopenMsChangePct": 66.45159219311996, "rows": 250, - "scenario": "orm indexed equality lookup", + "scenario": "table indexed equality lookup", "writeMsChangePct": 81.1194273934696, "writeSpeedup": 5.296607657679695, "writesPerSecondChangePct": 429.66076576796956 @@ -121,48 +121,48 @@ }, { "joinMs": 0.27866700000049605, - "label": "orm save() single transactions", + "label": "table upsert() single transactions", "reopenMs": 95.09941599999911, "rows": 250, "writeMs": 10266.896334000001, "writesPerSecond": 24.350104634065158, - "key": "orm-save-single" + "key": "table-upsert-single" }, { "joinMs": 0.3132499999992433, - "label": "orm insert() single transactions", + "label": "table insert() single transactions", "reopenMs": 145.7625420000004, "rows": 250, "writeMs": 10403.6615, "writesPerSecond": 24.030001360578677, - "key": "orm-insert" + "key": "table-insert" }, { "joinMs": 0.05662500000107684, - "label": "orm insertMany bulk transaction", + "label": "table insertMany bulk transaction", "reopenMs": 44.37445799999841, "rows": 250, "writeMs": 1868.581833999997, "writesPerSecond": 133.79130389212614, - "key": "orm-insert-many" + "key": "table-insert-many" }, { "joinMs": 0.05783299999893643, - "label": "orm saveMany bulk transaction", + "label": "table upsertMany bulk transaction", "reopenMs": 82.01083300000028, "rows": 250, "writeMs": 2459.1402080000007, "writesPerSecond": 101.66154788031506, - "key": "orm-save-many" + "key": "table-upsert-many" }, { "joinMs": 0.060459000000264496, - "label": "orm indexed equality lookup", + "label": "table indexed equality lookup", "reopenMs": 45.7231249999968, "rows": 250, "writeMs": 1015.2725830000018, "writesPerSecond": 246.23929000552906, - "key": "orm-indexed-lookup" + "key": "table-indexed-lookup" }, { "joinMs": 0.9968750000043656, diff --git a/site/benchmark.md b/site/benchmark.md index f78bd57..14d1fdf 100644 --- a/site/benchmark.md +++ b/site/benchmark.md @@ -2,6 +2,6 @@ | --- | ---: | ---: | ---: | ---: | ---: | | local plaintext transaction batch | 173.14 | 220.06 | +27.10% | +21.32% | +72.89% | | local encrypted transaction batch | 328.09 | 104.64 | -68.11% | -213.54% | +76.14% | -| orm indexed equality lookup | 46.49 | 246.24 | +429.66% | +81.12% | +95.35% | +| table indexed equality lookup | 46.49 | 246.24 | +429.66% | +81.12% | +95.35% | Runtime: v24.11.0 on darwin/arm64; repeat=1, warmup=0 diff --git a/site/index.html b/site/index.html index d4c68df..79f8d8e 100644 --- a/site/index.html +++ b/site/index.html @@ -33,7 +33,7 @@

GitHub Repo + DB Runtime

GitDB

- A local transaction runtime, repository API, page snapshots, + A local transaction runtime, table API, page snapshots, equality indexes, and compaction layer that uses GitHub repositories as durable audit storage.

@@ -54,7 +54,7 @@

GitDB

loading
-
Bulk vs save()
+
Bulk upsert
loading
@@ -168,8 +168,8 @@

GitHub stores history; GitDB owns the database runtime.

01 -

Repository API

-

Entities, repositories, `insertMany()`, `saveMany()`, and direct SQL queries.

+

Table API

+

Tables, `insertMany()`, `upsertMany()`, and direct SQL queries.

02 @@ -191,27 +191,35 @@

Remote audit

-

First-Party ORM

-

Use repositories for app code and bulk methods for throughput.

+

First-Party GitDB

+

Use GitDB tables for app code and bulk methods for throughput.

- 
import { LocalPlaintextStore, createGitDbDataSource, defineEntity } from "@3xhaust/gitdb"
+        
import { defineTable, GitDb, LocalPlaintextStore } from "@3xhaust/gitdb"
+import { z } from "zod"
 
-const Person = defineEntity({
-  tableName: "people",
-  primaryKey: "id",
+const PersonRow = z.object({
+  id: z.string(),
+  name: z.string(),
+  team_id: z.string(),
+})
+
+const Person = defineTable({
   columns: { id: "STRING", name: "STRING", team_id: "STRING" },
   indexes: [{ name: "people_team_id_idx", columns: ["team_id"] }],
+  name: "people",
+  primaryKey: "id",
+  row: PersonRow,
 })
 
-const dataSource = await createGitDbDataSource({
-  entities: [Person],
+const db = await GitDb.open({
   store: new LocalPlaintextStore({ root: ".gitdb" }),
-  synchronize: true,
+  syncSchema: true,
+  tables: [Person],
 })
 
-const people = dataSource.getRepository(Person)
+const people = db.table(Person)
 await people.insertMany([{ id: "p1", name: "Ada", team_id: "runtime" }])
-await people.find({ where: { team_id: "runtime" } })

+await people.select({ team_id: "runtime" })
diff --git a/src/errors.ts b/src/errors.ts index 0fcde0e..c00b174 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -37,6 +37,6 @@ export class ProtocolError extends Error { readonly name = "ProtocolError" } -export class GitDbOrmError extends Error { - readonly name = "GitDbOrmError" +export class GitDbSchemaError extends Error { + readonly name = "GitDbSchemaError" } diff --git a/src/gitdb/database.ts b/src/gitdb/database.ts new file mode 100644 index 0000000..8260a82 --- /dev/null +++ b/src/gitdb/database.ts @@ -0,0 +1,88 @@ +import { + GitDbEngine, + type GitDbEngineOptions, + type GitDbTransaction, + type SnapshotPolicy, +} from "../sql/engine.js" +import type { GitDbCompactionResult } from "../storage/store.js" +import type { GitDbDurabilityMode, SqlResult, SqlRow } from "../types.js" +import { createTableSql, type GitDbTableDefinition } from "./schema.js" +import { GitDbTable } from "./table.js" + +export type GitDbOpenOptions = { + readonly durability?: GitDbDurabilityMode + readonly snapshotPolicy?: SnapshotPolicy + readonly store: GitDbEngineOptions["store"] + readonly syncSchema?: boolean + readonly tables?: readonly GitDbTableDefinition[] +} + +export async function openGitDb(options: GitDbOpenOptions): Promise { + return await GitDb.open(options) +} + +export class GitDb { + readonly #engine: GitDbEngine + + constructor(engine: GitDbEngine) { + this.#engine = engine + } + + static async open(options: GitDbOpenOptions): Promise { + const engine = await GitDbEngine.open(engineOptions(options)) + const db = new GitDb(engine) + if (options.syncSchema === true) { + for (const table of options.tables ?? []) { + await db.createTable(table) + } + } + return db + } + + table(definition: GitDbTableDefinition): GitDbTable { + return new GitDbTable(this.#engine, definition) + } + + async createTable(definition: GitDbTableDefinition): Promise { + await this.#engine.execute(createTableSql(definition)) + } + + async execute(sql: string): Promise { + return await this.#engine.execute(sql) + } + + async query(sql: string): Promise { + return (await this.#engine.execute(sql)).rows + } + + async transaction(work: (transaction: GitDbTransaction) => Promise): Promise { + return await this.#engine.transaction(work) + } + + async compact(): Promise { + return await this.#engine.compact() + } + + getEngine(): GitDbEngine { + return this.#engine + } +} + +function engineOptions(options: GitDbOpenOptions): GitDbEngineOptions { + const durability = options.durability + const snapshotPolicy = options.snapshotPolicy + if (snapshotPolicy === undefined) { + if (durability === undefined) { + return { store: options.store } + } + return { durability, store: options.store } + } + if (durability === undefined) { + return { snapshotPolicy, store: options.store } + } + return { + durability, + snapshotPolicy, + store: options.store, + } +} diff --git a/src/gitdb/index.ts b/src/gitdb/index.ts new file mode 100644 index 0000000..e1927c7 --- /dev/null +++ b/src/gitdb/index.ts @@ -0,0 +1,9 @@ +export { GitDb, type GitDbOpenOptions, openGitDb } from "./database.js" +export { + defineTable, + type GitDbColumnType, + type GitDbTableDefinition, + type GitDbTableIndexDefinition, + type GitDbWhere, +} from "./schema.js" +export { type GitDbInsertResult, GitDbTable, type GitDbUpsertResult } from "./table.js" diff --git a/src/gitdb/schema.ts b/src/gitdb/schema.ts new file mode 100644 index 0000000..4ae4164 --- /dev/null +++ b/src/gitdb/schema.ts @@ -0,0 +1,165 @@ +import { type ZodType, z } from "zod" +import { GitDbSchemaError } from "../errors.js" +import { sqlLiteral } from "../sql/rows.js" +import type { JsonPrimitive, SqlRow } from "../types.js" + +const GitDbColumnTypes = ["STRING", "INT", "FLOAT", "BOOL"] as const +const SqlIdentifierSchema = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/) + +const TableDefinitionMetadataSchema = z + .object({ + columns: z.record(z.string().min(1), z.enum(GitDbColumnTypes)), + indexes: z + .array( + z.object({ + columns: z.array(z.string().min(1)).min(1), + name: SqlIdentifierSchema, + }), + ) + .optional(), + name: SqlIdentifierSchema, + primaryKey: z.string().min(1), + }) + .superRefine((definition, context) => { + if (!(definition.primaryKey in definition.columns)) { + context.addIssue({ + code: "custom", + message: "primaryKey must exist in columns", + path: ["primaryKey"], + }) + } + definition.indexes?.forEach((index, indexOffset) => { + index.columns.forEach((column, columnOffset) => { + if (!(column in definition.columns)) { + context.addIssue({ + code: "custom", + message: `indexed column ${column} does not exist`, + path: ["indexes", indexOffset, "columns", columnOffset], + }) + } + }) + }) + }) + +export type GitDbColumnType = (typeof GitDbColumnTypes)[number] + +export type GitDbTableIndexDefinition = { + readonly columns: readonly ColumnName[] + readonly name: string +} + +export type GitDbTableDefinition = { + readonly columns: Readonly, GitDbColumnType>> + readonly indexes?: readonly GitDbTableIndexDefinition>[] + readonly name: string + readonly primaryKey: Extract + readonly row: ZodType +} + +export type GitDbWhere = Partial< + Record, JsonPrimitive> +> + +export function defineTable( + definition: GitDbTableDefinition, +): GitDbTableDefinition { + TableDefinitionMetadataSchema.parse({ + columns: definition.columns, + indexes: definition.indexes, + name: definition.name, + primaryKey: definition.primaryKey, + }) + return definition +} + +export function createTableSql(definition: GitDbTableDefinition): string { + const columns = Object.entries(definition.columns) + .map(([name, type]) => `${identifier(name)} ${type}`) + .join(", ") + return `CREATE TABLE IF NOT EXISTS ${identifier(definition.name)} (${columns})` +} + +export function insertSql( + definition: GitDbTableDefinition, + row: Row, +): string { + const columns = Object.keys(definition.columns) + const names = columns.map(identifier).join(", ") + const values = columns.map((column) => sqlLiteral(valueForKey(row, column))).join(", ") + return `INSERT INTO ${identifier(definition.name)} (${names}) VALUES (${values})` +} + +export function selectSql( + definition: GitDbTableDefinition, + where: GitDbWhere, +): string { + const clause = whereClause(where) + return `SELECT * FROM ${identifier(definition.name)}${clause}` +} + +export function deleteSql( + definition: GitDbTableDefinition, + where: GitDbWhere, +): string { + const clause = whereClause(where) + if (clause.length === 0) { + throw new GitDbSchemaError("deleteWhere requires a non-empty where clause") + } + return `DELETE FROM ${identifier(definition.name)}${clause}` +} + +export function parseTableRow( + definition: GitDbTableDefinition, + row: SqlRow, +): Row { + return definition.row.parse(projectSqlRow(definition, row)) +} + +export function primitiveValueForKey(row: Row, key: string): JsonPrimitive { + const value: unknown = Reflect.get(row, key) + if (isJsonPrimitive(value)) { + return value + } + throw new GitDbSchemaError(`table value for ${key} must be a JSON primitive`) +} + +function whereClause(where: GitDbWhere): string { + if (Object.keys(where).length === 0) { + return "" + } + const parts = Object.entries(where).map( + ([column, value]) => `${identifier(column)} = ${sqlLiteral(value)}`, + ) + return ` WHERE ${parts.join(" AND ")}` +} + +function identifier(value: string): string { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) { + throw new GitDbSchemaError(`invalid SQL identifier: ${value}`) + } + return value +} + +function valueForKey(row: Row, key: string): JsonPrimitive { + return primitiveValueForKey(row, key) +} + +function projectSqlRow( + definition: GitDbTableDefinition, + row: SqlRow, +): SqlRow { + const projected: Record = {} + for (const column of Object.keys(definition.columns)) { + projected[column] = row[column] ?? null + } + return projected +} + +function isJsonPrimitive(value: unknown): value is JsonPrimitive { + return ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) +} diff --git a/src/gitdb/table.ts b/src/gitdb/table.ts new file mode 100644 index 0000000..113153a --- /dev/null +++ b/src/gitdb/table.ts @@ -0,0 +1,89 @@ +import type { GitDbEngine } from "../sql/engine.js" +import type { SqlResult } from "../types.js" +import { + deleteSql, + type GitDbTableDefinition, + type GitDbWhere, + insertSql, + parseTableRow, + primitiveValueForKey, + selectSql, +} from "./schema.js" + +export type GitDbInsertResult = { + readonly inserted: number +} + +export type GitDbUpsertResult = { + readonly upserted: number +} + +export class GitDbTable { + readonly #engine: GitDbEngine + readonly #definition: GitDbTableDefinition + + constructor(engine: GitDbEngine, definition: GitDbTableDefinition) { + this.#engine = engine + this.#definition = definition + } + + async insert(row: Row): Promise { + return await this.insertMany([row]) + } + + async insertMany(rows: readonly Row[]): Promise { + if (rows.length === 0) { + return { inserted: 0 } + } + await this.#engine.transaction(async (transaction) => { + for (const row of rows) { + await transaction.execute(insertSql(this.#definition, row)) + } + }) + return { inserted: rows.length } + } + + async upsert(row: Row): Promise { + return await this.upsertMany([row]) + } + + async upsertMany(rows: readonly Row[]): Promise { + if (rows.length === 0) { + return { upserted: 0 } + } + await this.#engine.transaction(async (transaction) => { + for (const row of rows) { + await transaction.execute( + deleteSql(this.#definition, primaryKeyWhere(this.#definition, row)), + ) + await transaction.execute(insertSql(this.#definition, row)) + } + }) + return { upserted: rows.length } + } + + async select(where: GitDbWhere = {}): Promise { + const result = await this.#engine.execute(selectSql(this.#definition, where)) + return result.rows.map((row) => parseTableRow(this.#definition, row)) + } + + async first(where: GitDbWhere): Promise { + const rows = await this.select(where) + return rows[0] ?? null + } + + async deleteWhere(where: GitDbWhere): Promise { + const result: SqlResult = await this.#engine.execute(deleteSql(this.#definition, where)) + return result.rowCount + } +} + +function primaryKeyWhere( + definition: GitDbTableDefinition, + row: Row, +): GitDbWhere { + const value = primitiveValueForKey(row, definition.primaryKey) + const where: GitDbWhere = {} + where[definition.primaryKey] = value + return where +} diff --git a/src/github/github-encrypted-store.ts b/src/github/github-encrypted-store.ts index 676340c..152b9a8 100644 --- a/src/github/github-encrypted-store.ts +++ b/src/github/github-encrypted-store.ts @@ -80,8 +80,11 @@ export class GitHubEncryptedStore implements GitDbStore { } const metadata = normalizeManifestMetadata(parsed.metadata) return metadata === undefined ? manifest : { ...manifest, metadata } - } catch (error) { - throw manifestReadError(path, error) + } catch (error: unknown) { + if (error instanceof Error) { + throw manifestReadError(path, error) + } + throw manifestReadError(path, new GitDbStorageError(String(error))) } } @@ -173,6 +176,9 @@ export class GitHubEncryptedStore implements GitDbStore { } const current = await this.readManifest() const currentSequence = current?.sequence ?? null + if (currentSequence === null && compareAndSwap.expectedSequence === 0) { + return + } if (currentSequence !== compareAndSwap.expectedSequence) { throw new GitDbStorageError( `manifest compare-and-swap failed: expected sequence ${compareAndSwap.expectedSequence}, got ${currentSequence}`, @@ -186,9 +192,8 @@ export class GitHubEncryptedStore implements GitDbStore { } } -function manifestReadError(path: string, error: unknown): GitDbStorageError { - const detail = error instanceof Error ? error.message : String(error) - return new GitDbStorageError(`failed to read GitHub manifest at ${path}: ${detail}`) +function manifestReadError(path: string, error: Error): GitDbStorageError { + return new GitDbStorageError(`failed to read GitHub manifest at ${path}: ${error.message}`) } function normalizeManifestMetadata( diff --git a/src/github/github-plaintext-store.ts b/src/github/github-plaintext-store.ts index 1859641..1426b1f 100644 --- a/src/github/github-plaintext-store.ts +++ b/src/github/github-plaintext-store.ts @@ -195,6 +195,9 @@ export class GitHubPlaintextStore implements GitDbStore { } const current = await this.readManifest() const currentSequence = current?.sequence ?? null + if (currentSequence === null && compareAndSwap.expectedSequence === 0) { + return + } if (currentSequence !== compareAndSwap.expectedSequence) { throw new GitDbStorageError( `manifest compare-and-swap failed: expected sequence ${compareAndSwap.expectedSequence}, got ${currentSequence}`, diff --git a/src/index.ts b/src/index.ts index 3892180..e45a791 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,20 +1,20 @@ export { generateKey } from "./cli/keygen.js" export { createAesGcmCipher } from "./crypto/aes-gcm.js" -export { GitHubEncryptedStore } from "./github/github-encrypted-store.js" export { - createGitDbDataSource, - defineEntity, - type EntityDefinition, - type FindOptions, + defineTable, + GitDb, type GitDbColumnType, - GitDbDataSource, - type GitDbDataSourceOptions, - type GitDbEntityDefinition, - type GitDbEntityIndexDefinition, type GitDbInsertResult, - GitDbRepository, - type GitDbSaveManyResult, -} from "./orm/index.js" + type GitDbOpenOptions, + GitDbTable, + type GitDbTableDefinition, + type GitDbTableIndexDefinition, + type GitDbUpsertResult, + type GitDbWhere, + openGitDb, +} from "./gitdb/index.js" +export { GitHubEncryptedStore } from "./github/github-encrypted-store.js" +export { GitHubPlaintextStore } from "./github/github-plaintext-store.js" export { GitDbEngine, type GitDbEngineOptions, diff --git a/src/orm/index.ts b/src/orm/index.ts deleted file mode 100644 index 658ca6e..0000000 --- a/src/orm/index.ts +++ /dev/null @@ -1,270 +0,0 @@ -import { z } from "zod" -import { GitDbOrmError } from "../errors.js" -import { GitDbEngine, type GitDbTransaction, type SnapshotPolicy } from "../sql/engine.js" -import { sqlLiteral } from "../sql/rows.js" -import type { GitDbStore } from "../storage/store.js" -import type { JsonPrimitive, SqlRow } from "../types.js" - -const GitDbColumnTypes = ["STRING", "INT", "FLOAT", "BOOL"] as const - -export type GitDbColumnType = (typeof GitDbColumnTypes)[number] - -export type GitDbEntityIndexDefinition = { - readonly columns: readonly ColumnName[] - readonly name: string -} - -export type GitDbEntityDefinition = { - readonly columns: Readonly> - readonly indexes?: readonly GitDbEntityIndexDefinition[] - readonly primaryKey: string - readonly tableName: string -} - -export type EntityDefinition = Omit< - GitDbEntityDefinition, - "columns" | "indexes" | "primaryKey" -> & { - readonly columns: { readonly [Key in Extract]: GitDbColumnType } - readonly indexes?: readonly GitDbEntityIndexDefinition>[] - readonly primaryKey: Extract -} - -export type FindOptions = { - readonly where?: Partial, JsonPrimitive>> -} - -export type GitDbDataSourceOptions = { - readonly entities: readonly GitDbEntityDefinition[] - readonly snapshotPolicy?: SnapshotPolicy - readonly store: GitDbStore - readonly synchronize?: boolean -} - -export type GitDbInsertResult = { - readonly inserted: number -} - -export type GitDbSaveManyResult = { - readonly saved: number -} - -const SqlIdentifierSchema = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/) - -const EntityDefinitionSchema = z - .object({ - columns: z.record(z.string().min(1), z.enum(GitDbColumnTypes)), - indexes: z - .array( - z.object({ - columns: z.array(z.string().min(1)).min(1), - name: SqlIdentifierSchema, - }), - ) - .optional(), - primaryKey: z.string().min(1), - tableName: SqlIdentifierSchema, - }) - .superRefine((definition, context) => { - if (!(definition.primaryKey in definition.columns)) { - context.addIssue({ - code: "custom", - message: "primaryKey must exist in columns", - path: ["primaryKey"], - }) - } - definition.indexes?.forEach((index, indexOffset) => { - index.columns.forEach((column, columnOffset) => { - if (!(column in definition.columns)) { - context.addIssue({ - code: "custom", - message: `indexed column ${column} does not exist`, - path: ["indexes", indexOffset, "columns", columnOffset], - }) - } - }) - }) - }) - -export function defineEntity( - definition: EntityDefinition, -): EntityDefinition { - EntityDefinitionSchema.parse(definition) - return definition -} - -export async function createGitDbDataSource( - options: GitDbDataSourceOptions, -): Promise { - const engine = - options.snapshotPolicy === undefined - ? await GitDbEngine.open({ store: options.store }) - : await GitDbEngine.open({ - snapshotPolicy: options.snapshotPolicy, - store: options.store, - }) - const dataSource = new GitDbDataSource(engine) - if (options.synchronize === true) { - for (const entity of options.entities) { - await dataSource.synchronize(entity) - } - } - return dataSource -} - -export class GitDbDataSource { - readonly #engine: GitDbEngine - - constructor(engine: GitDbEngine) { - this.#engine = engine - } - - async query(sql: string): Promise { - return (await this.#engine.execute(sql)).rows - } - - getRepository(entity: EntityDefinition): GitDbRepository { - return new GitDbRepository(this.#engine, entity) - } - - async synchronize(entity: GitDbEntityDefinition): Promise { - await this.#engine.execute(createTableSql(entity)) - } - - async transaction(work: (transaction: GitDbTransaction) => Promise): Promise { - return await this.#engine.transaction(work) - } -} - -export class GitDbRepository { - readonly #engine: GitDbEngine - readonly #entity: EntityDefinition - - constructor(engine: GitDbEngine, entity: EntityDefinition) { - this.#engine = engine - this.#entity = entity - } - - async save(row: Row): Promise { - await this.saveMany([row]) - } - - async saveMany(rows: readonly Row[]): Promise { - if (rows.length === 0) { - return { saved: 0 } - } - await this.#engine.transaction(async (transaction) => { - for (const row of rows) { - await transaction.execute(deleteSql(this.#entity, primaryKeyWhere(this.#entity, row))) - await transaction.execute(insertSql(this.#entity, row)) - } - }) - return { saved: rows.length } - } - - async insert(row: Row): Promise { - return await this.insertMany([row]) - } - - async insertMany(rows: readonly Row[]): Promise { - if (rows.length === 0) { - return { inserted: 0 } - } - await this.#engine.transaction(async (transaction) => { - for (const row of rows) { - await transaction.execute(insertSql(this.#entity, row)) - } - }) - return { inserted: rows.length } - } - - async find(options: FindOptions = {}): Promise { - const result = await this.#engine.execute(selectSql(this.#entity, options.where ?? {})) - return result.rows.map((row) => row as Row) - } - - async findOne(options: FindOptions): Promise { - const rows = await this.find(options) - return rows[0] ?? null - } - - async delete(where: FindOptions["where"]): Promise { - if (where === undefined || Object.keys(where).length === 0) { - throw new GitDbOrmError("delete requires a non-empty where clause") - } - const result = await this.#engine.execute(deleteSql(this.#entity, where)) - return result.rowCount - } -} - -function createTableSql(entity: GitDbEntityDefinition): string { - const columns = Object.entries(entity.columns) - .map(([name, type]) => `${identifier(name)} ${type}`) - .join(", ") - return `CREATE TABLE IF NOT EXISTS ${identifier(entity.tableName)} (${columns})` -} - -function insertSql(entity: EntityDefinition, row: Row): string { - const columns = Object.keys(entity.columns) - const names = columns.map(identifier).join(", ") - const values = columns.map((column) => sqlLiteral(valueForKey(row, column))).join(", ") - return `INSERT INTO ${identifier(entity.tableName)} (${names}) VALUES (${values})` -} - -function selectSql( - entity: EntityDefinition, - where: FindOptions["where"], -): string { - const clause = whereClause(where) - return `SELECT * FROM ${identifier(entity.tableName)}${clause}` -} - -function deleteSql( - entity: EntityDefinition, - where: FindOptions["where"], -): string { - const clause = whereClause(where) - if (clause.length === 0) { - throw new GitDbOrmError("delete requires a non-empty where clause") - } - return `DELETE FROM ${identifier(entity.tableName)}${clause}` -} - -function whereClause(where: FindOptions["where"]): string { - if (where === undefined || Object.keys(where).length === 0) { - return "" - } - const parts = Object.entries(where).map( - ([column, value]) => `${identifier(column)} = ${sqlLiteral(value)}`, - ) - return ` WHERE ${parts.join(" AND ")}` -} - -function identifier(value: string): string { - if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) { - throw new GitDbOrmError(`invalid SQL identifier: ${value}`) - } - return value -} - -function valueForKey(row: Row, key: string): JsonPrimitive { - const value: unknown = Reflect.get(row, key) - if ( - value === null || - typeof value === "string" || - typeof value === "number" || - typeof value === "boolean" - ) { - return value - } - throw new GitDbOrmError(`entity value for ${key} must be a JSON primitive`) -} - -function primaryKeyWhere( - entity: EntityDefinition, - row: Row, -): Partial, JsonPrimitive>> { - const where: Partial, JsonPrimitive>> = {} - where[entity.primaryKey] = valueForKey(row, entity.primaryKey) - return where -} diff --git a/tests/benchmark-report.test.ts b/tests/benchmark-report.test.ts index 97aaa88..d6688d5 100644 --- a/tests/benchmark-report.test.ts +++ b/tests/benchmark-report.test.ts @@ -9,7 +9,7 @@ const ProbeSchema = z.object({ comparisonCount: z.number(), headline: z.string(), metadataRepeatCount: z.number(), - markdownIncludesOrm: z.boolean(), + markdownIncludesTable: z.boolean(), runtimeIncludesNode: z.boolean(), scenarioKeys: z.array(z.string()), }) @@ -22,11 +22,11 @@ describe("benchmark report parser", () => { // When/Then: comparable scenarios become site-ready evidence. expect(probe).toEqual({ comparisonCount: 3, - headline: "Raw engine is 3.00x of ORM writes/s (save() transaction overhead)", + headline: "Raw SQL is 3.00x of table upsert writes/s", metadataRepeatCount: 2, - markdownIncludesOrm: true, + markdownIncludesTable: true, runtimeIncludesNode: true, - scenarioKeys: ["local-plaintext", "local-encrypted", "orm-save-single"], + scenarioKeys: ["local-plaintext", "local-encrypted", "table-upsert-single"], }) }) @@ -60,7 +60,7 @@ async function runBenchmarkReportProbe(): Promise> { | --- | ---: | ---: | ---: | ---: | ---: | | local plaintext throttled visible snapshots | 250 | 100 | 2500 | 5 | 20 | | local encrypted mutation log | 250 | 50 | 5000 | 4 | 10 | -| orm save() single transactions | 250 | 300 | 833.33 | 8 | 30 | +| table upsert() single transactions | 250 | 300 | 833.33 | 8 | 30 | \`) const current = parseBenchmarkEvidence(JSON.stringify({ metadata: { @@ -89,7 +89,7 @@ async function runBenchmarkReportProbe(): Promise> { }, { joinMs: 7, - label: 'orm save() single transactions', + label: 'table upsert() single transactions', reopenMs: 28, rows: 250, writeMs: 270, @@ -111,7 +111,7 @@ async function runBenchmarkReportProbe(): Promise> { comparisonCount: evidence.comparisons.length, headline: evidence.headline, metadataRepeatCount: evidence.current.metadata.repeatCount, - markdownIncludesOrm: markdown.includes('orm save() single transactions'), + markdownIncludesTable: markdown.includes('table upsert() single transactions'), runtimeIncludesNode: markdown.includes('Runtime: v-test on darwin/arm64; repeat=2, warmup=1'), scenarioKeys: evidence.scenarios.map((scenario) => scenario.key), })) diff --git a/tests/benchmark-script.test.ts b/tests/benchmark-script.test.ts index 4552431..9e1bd14 100644 --- a/tests/benchmark-script.test.ts +++ b/tests/benchmark-script.test.ts @@ -100,11 +100,11 @@ describe("benchmark evaluator", () => { [ "checked scenarios: local-plaintext", "local-encrypted", - "orm-save-single", - "orm-insert", - "orm-insert-many", - "orm-save-many", - "orm-indexed-lookup", + "table-upsert-single", + "table-insert", + "table-insert-many", + "table-upsert-many", + "table-indexed-lookup", "reopen", "compaction", ].join(", "), @@ -126,11 +126,11 @@ describe("benchmark evaluator", () => { scenarios: [ benchmarkRow("local plaintext throttled visible snapshots"), benchmarkRow("local encrypted mutation log"), - benchmarkRow("orm save() single transactions"), - benchmarkRow("orm insert() single transactions"), - benchmarkRow("orm insertMany bulk transaction"), - benchmarkRow("orm saveMany bulk transaction"), - benchmarkRow("orm indexed equality lookup"), + benchmarkRow("table upsert() single transactions"), + benchmarkRow("table insert() single transactions"), + benchmarkRow("table insertMany bulk transaction"), + benchmarkRow("table upsertMany bulk transaction"), + benchmarkRow("table indexed equality lookup"), benchmarkRow("local plaintext reopen checkpoint"), benchmarkRow("local plaintext compaction"), ], @@ -156,11 +156,11 @@ function completeBenchmarkEvidence() { results: [ benchmarkRow("local plaintext throttled visible snapshots"), benchmarkRow("local encrypted mutation log"), - benchmarkRow("orm save() single transactions"), - benchmarkRow("orm insert() single transactions"), - benchmarkRow("orm insertMany bulk transaction"), - benchmarkRow("orm saveMany bulk transaction"), - benchmarkRow("orm indexed equality lookup"), + benchmarkRow("table upsert() single transactions"), + benchmarkRow("table insert() single transactions"), + benchmarkRow("table insertMany bulk transaction"), + benchmarkRow("table upsertMany bulk transaction"), + benchmarkRow("table indexed equality lookup"), benchmarkRow("local plaintext reopen checkpoint"), benchmarkRow("local plaintext compaction"), ], diff --git a/tests/e2e.test.ts b/tests/e2e.test.ts index 877f89f..9b207b1 100644 --- a/tests/e2e.test.ts +++ b/tests/e2e.test.ts @@ -2,101 +2,105 @@ import { mkdtemp } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" import { describe, expect, it } from "vitest" +import { z } from "zod" import { createAesGcmCipher } from "../src/crypto/aes-gcm.js" -import { createGitDbDataSource, defineEntity } from "../src/orm/index.js" +import { defineTable, GitDb } from "../src/gitdb/index.js" import { LocalEncryptedStore } from "../src/storage/local-encrypted-store.js" -type Team = { - readonly id: string - readonly name: string -} +const TeamRow = z.object({ + id: z.string(), + name: z.string(), +}) + +const PersonRow = z.object({ + id: z.string(), + name: z.string(), + team_id: z.string(), +}) -type Person = { - readonly id: string - readonly name: string - readonly team_id: string -} +type Team = z.infer +type Person = z.infer -const TeamEntity = defineEntity({ +const TeamTable = defineTable({ columns: { id: "STRING", name: "STRING", }, + name: "teams", primaryKey: "id", - tableName: "teams", + row: TeamRow, }) -const PersonEntity = defineEntity({ +const PersonTable = defineTable({ columns: { id: "STRING", name: "STRING", team_id: "STRING", }, + name: "people", primaryKey: "id", - tableName: "people", + row: PersonRow, }) describe("GitDB first-party runtime", () => { - it("writes through repositories, joins through the engine, and reopens encrypted state", async () => { - // Given: a GitDB data source backed by encrypted local storage. + it("writes through tables, joins through the engine, and reopens encrypted state", async () => { + // Given: a GitDB database backed by encrypted local storage. const root = await mkdtemp(join(tmpdir(), "gitdb-runtime-")) const key = Buffer.alloc(32, 3).toString("base64url") const cipher = createAesGcmCipher(key) - const entities = [TeamEntity, PersonEntity] - const dataSource = await createGitDbDataSource({ - entities, + const tables = [TeamTable, PersonTable] + const db = await GitDb.open({ store: new LocalEncryptedStore({ cipher, root }), - synchronize: true, + syncSchema: true, + tables, }) - // When: an app uses first-party repositories and SQL on the same runtime. - await dataSource.getRepository(TeamEntity).save({ id: "t1", name: "Storage" }) - await dataSource.getRepository(PersonEntity).save({ id: "p1", name: "Lin", team_id: "t1" }) - const joined = await dataSource.query( + // When: an app uses first-party table handles and SQL on the same runtime. + await db.table(TeamTable).upsert({ id: "t1", name: "Storage" }) + await db.table(PersonTable).upsert({ id: "p1", name: "Lin", team_id: "t1" }) + const joined = await db.query( "SELECT people.name AS person, teams.name AS team FROM people JOIN teams ON people.team_id = teams.id", ) - const reopened = await createGitDbDataSource({ - entities, + const reopened = await GitDb.open({ store: new LocalEncryptedStore({ cipher: createAesGcmCipher(key), root }), - synchronize: false, + tables, }) - // Then: the runtime behaves as one database across repository, query, and reopen paths. + // Then: the runtime behaves as one database across table, query, and reopen paths. expect(joined).toEqual([{ person: "Lin", team: "Storage" }]) - await expect(reopened.getRepository(PersonEntity).find()).resolves.toEqual([ + await expect(reopened.table(PersonTable).select()).resolves.toEqual([ { id: "p1", name: "Lin", team_id: "t1" }, ]) }) - it("bulk repositories survive encrypted reopen", async () => { - // Given: encrypted local storage and synchronized repositories. + it("bulk table writes survive encrypted reopen", async () => { + // Given: encrypted local storage and synchronized tables. const root = await mkdtemp(join(tmpdir(), "gitdb-runtime-bulk-")) const key = Buffer.alloc(32, 17).toString("base64url") const cipher = createAesGcmCipher(key) - const entities = [TeamEntity, PersonEntity] - const dataSource = await createGitDbDataSource({ - entities, + const tables = [TeamTable, PersonTable] + const db = await GitDb.open({ store: new LocalEncryptedStore({ cipher, root }), - synchronize: true, + syncSchema: true, + tables, }) - // When: repositories use bulk APIs and the database reopens. - await dataSource.getRepository(TeamEntity).insertMany([ + // When: table handles use bulk APIs and the database reopens. + await db.table(TeamTable).insertMany([ { id: "t1", name: "Storage" }, { id: "t2", name: "Runtime" }, ]) - await dataSource.getRepository(PersonEntity).saveMany([ + await db.table(PersonTable).upsertMany([ { id: "p1", name: "Lin", team_id: "t1" }, { id: "p2", name: "Ada", team_id: "t2" }, ]) - const reopened = await createGitDbDataSource({ - entities, + const reopened = await GitDb.open({ store: new LocalEncryptedStore({ cipher: createAesGcmCipher(key), root }), - synchronize: false, + tables, }) // Then: bulk-written rows survive encrypted replay. - await expect(reopened.getRepository(PersonEntity).find()).resolves.toEqual([ + await expect(reopened.table(PersonTable).select()).resolves.toEqual([ { id: "p1", name: "Lin", team_id: "t1" }, { id: "p2", name: "Ada", team_id: "t2" }, ]) diff --git a/tests/gitdb.test.ts b/tests/gitdb.test.ts new file mode 100644 index 0000000..a780d5d --- /dev/null +++ b/tests/gitdb.test.ts @@ -0,0 +1,175 @@ +import { mkdtemp } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { z } from "zod" +import { defineTable, GitDb } from "../src/gitdb/index.js" +import { LocalPlaintextStore } from "../src/storage/local-plaintext-store.js" +import { CountingBatchStore } from "./sql-engine-fixtures.js" + +const PersonRow = z.object({ + id: z.string(), + name: z.string(), + team_id: z.string(), +}) + +const LoosePersonRow = z.object({ + id: z.string(), + name: z.union([z.string(), z.object({ nested: z.boolean() })]), + team_id: z.string(), +}) + +const StringRecordRow = z.record(z.string(), z.string()) + +type Person = z.infer +type LoosePerson = z.infer + +const PersonTable = defineTable({ + columns: { + id: "STRING", + name: "STRING", + team_id: "STRING", + }, + name: "people", + primaryKey: "id", + row: PersonRow, +}) + +const LoosePersonTable = defineTable({ + columns: { + id: "STRING", + name: "STRING", + team_id: "STRING", + }, + name: "people", + primaryKey: "id", + row: LoosePersonRow, +}) + +describe("GitDB table API", () => { + it("validates table and secondary index metadata", () => { + // Given: table metadata with a declared equality-only secondary index. + const indexed = defineTable({ + columns: { + id: "STRING", + name: "STRING", + team_id: "STRING", + }, + indexes: [{ columns: ["team_id"], name: "people_team_id_idx" }], + name: "people", + primaryKey: "id", + row: PersonRow, + }) + + // When/Then: valid index metadata is preserved for the local index runtime. + expect(indexed.indexes).toEqual([{ columns: ["team_id"], name: "people_team_id_idx" }]) + expect(() => + defineTable>({ + columns: { + id: "STRING", + name: "STRING", + team_id: "STRING", + }, + indexes: [{ columns: ["missing"], name: "people_missing_idx" }], + name: "people", + primaryKey: "id", + row: StringRecordRow, + }), + ).toThrow("indexed column missing does not exist") + }) + + it("inserts, selects, and deletes rows through a GitDB-native table handle", async () => { + // Given: a GitDB database synchronized from table metadata. + const root = await mkdtemp(join(tmpdir(), "gitdb-table-")) + const db = await GitDb.open({ + store: new LocalPlaintextStore({ root }), + syncSchema: true, + tables: [PersonTable], + }) + const people = db.table(PersonTable) + + // When: callers use table operations over the local runtime. + await people.upsert({ id: "p1", name: "Lin", team_id: "storage" }) + await people.upsert({ id: "p2", name: "Ada", team_id: "runtime" }) + const found = await people.select({ team_id: "storage" }) + const one = await people.first({ id: "p2" }) + await people.deleteWhere({ id: "p1" }) + + // Then: the table exposes RDB-like row operations without a repository layer. + await expect(people.select()).resolves.toEqual([{ id: "p2", name: "Ada", team_id: "runtime" }]) + expect(found).toEqual([{ id: "p1", name: "Lin", team_id: "storage" }]) + expect(one).toEqual({ id: "p2", name: "Ada", team_id: "runtime" }) + }) + + it("inserts many rows in one table transaction", async () => { + // Given: a synchronized table backed by a counting v2 store. + const root = await mkdtemp(join(tmpdir(), "gitdb-table-bulk-")) + const store = new CountingBatchStore(new LocalPlaintextStore({ root })) + const db = await GitDb.open({ store, syncSchema: true, tables: [PersonTable] }) + const people = db.table(PersonTable) + store.resetCounts() + + // When: callers insert multiple rows through the table bulk API. + const result = await people.insertMany([ + { id: "p1", name: "Lin", team_id: "storage" }, + { id: "p2", name: "Ada", team_id: "runtime" }, + ]) + + // Then: rows are visible and persistence used one engine batch boundary. + expect(result).toEqual({ inserted: 2 }) + await expect(people.select()).resolves.toEqual([ + { id: "p1", name: "Lin", team_id: "storage" }, + { id: "p2", name: "Ada", team_id: "runtime" }, + ]) + expect(store.commitBatchCalls).toBe(1) + expect(store.writeManifestCalls).toBeLessThanOrEqual(3) + }) + + it("treats empty and invalid bulk inserts as atomic table operations", async () => { + // Given: a synchronized table with no rows. + const root = await mkdtemp(join(tmpdir(), "gitdb-table-bulk-invalid-")) + const store = new CountingBatchStore(new LocalPlaintextStore({ root })) + const db = await GitDb.open({ store, syncSchema: true, tables: [PersonTable] }) + const people = db.table(PersonTable) + const loosePeople = db.table(LoosePersonTable) + store.resetCounts() + + // When/Then: empty insertMany is a no-op and does not persist. + await expect(people.insertMany([])).resolves.toEqual({ inserted: 0 }) + expect(store.commitBatchCalls).toBe(0) + + // When/Then: invalid rows reject and leave no partial rows visible. + await expect( + loosePeople.insertMany([ + { id: "p1", name: "Lin", team_id: "storage" }, + { id: "bad", name: { nested: true }, team_id: "runtime" }, + ]), + ).rejects.toThrow("table value for name") + await expect(people.select()).resolves.toEqual([]) + }) + + it("upserts many rows in one table transaction", async () => { + // Given: existing rows and a counting store. + const root = await mkdtemp(join(tmpdir(), "gitdb-table-upsert-many-")) + const store = new CountingBatchStore(new LocalPlaintextStore({ root })) + const db = await GitDb.open({ store, syncSchema: true, tables: [PersonTable] }) + const people = db.table(PersonTable) + await people.insertMany([{ id: "p1", name: "Lin", team_id: "storage" }]) + const updates = Array.from({ length: 100 }, (_, index) => ({ + id: `p${index + 1}`, + name: `Person ${index + 1}`, + team_id: index % 2 === 0 ? "storage" : "runtime", + })) + store.resetCounts() + + // When: upsertMany mixes overwrite and insert rows. + const result = await people.upsertMany(updates) + + // Then: upsertMany preserves primary-key overwrite semantics inside one batch. + const rows = await db.query("SELECT * FROM people WHERE id = 'p1'") + expect(result).toEqual({ upserted: 100 }) + expect(rows).toEqual([{ id: "p1", name: "Person 1", team_id: "storage" }]) + expect(store.commitBatchCalls).toBe(1) + expect(store.writeManifestCalls).toBeLessThanOrEqual(3) + }) +}) diff --git a/tests/github-tree-store.test.ts b/tests/github-tree-store.test.ts index bbc4ab8..d12adb4 100644 --- a/tests/github-tree-store.test.ts +++ b/tests/github-tree-store.test.ts @@ -153,6 +153,25 @@ describe("GitHub tree commit stores", () => { } }) + it("allows the initial encrypted compare-and-swap while GitHub contents catches up", async () => { + // Given: the manifest was just bootstrapped, but GitHub Contents still returns 404. + octokit.getContent.mockRejectedValue(new FakeGitHubStatusError(404)) + const store = new GitHubEncryptedStore( + config, + createAesGcmCipher(Buffer.alloc(32, 31).toString("base64url")), + ) + + // When: the first durable batch checks the initial sequence. + const result = await store.commitBatch({ + ...batch(), + compareAndSwap: { expectedSequence: 0 }, + }) + + // Then: the missing manifest is treated as the empty initial sequence. + expect(result.segments).toEqual(manifest.logSegments) + expect(octokit.createTree).toHaveBeenCalledTimes(1) + }) + it("rejects stale manifest compare-and-swap before creating a tree", async () => { // Given: the remote manifest has advanced beyond the caller's expected sequence. octokit.getContent.mockResolvedValue({ diff --git a/tests/index-runtime.test.ts b/tests/index-runtime.test.ts index c9c561b..ffe4295 100644 --- a/tests/index-runtime.test.ts +++ b/tests/index-runtime.test.ts @@ -2,42 +2,46 @@ import { mkdtemp, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" import { describe, expect, it } from "vitest" -import { createGitDbDataSource, defineEntity, GitDbDataSource } from "../src/orm/index.js" +import { z } from "zod" +import { defineTable, GitDb } from "../src/gitdb/index.js" import { GitDbEngine } from "../src/sql/engine.js" import { LocalPlaintextStore } from "../src/storage/local-plaintext-store.js" -type Person = { - readonly id: string - readonly name: string - readonly team_id: string -} +const PersonRow = z.object({ + id: z.string(), + name: z.string(), + team_id: z.string(), +}) + +type Person = z.infer -const PersonEntity = defineEntity({ +const PersonTable = defineTable({ columns: { id: "STRING", name: "STRING", team_id: "STRING", }, indexes: [{ columns: ["team_id"], name: "people_team_id_idx" }], + name: "people", primaryKey: "id", - tableName: "people", + row: PersonRow, }) describe("index runtime", () => { it("uses primary and secondary indexes for equality lookup", async () => { - // Given: a repository has rows addressable by primary key and secondary equality metadata. + // Given: a table has rows addressable by primary key and secondary equality metadata. const root = await mkdtemp(join(tmpdir(), "gitdb-index-runtime-")) const store = new LocalPlaintextStore({ root }) const engine = await GitDbEngine.open({ store }) - const dataSource = new GitDbDataSource(engine) - await dataSource.synchronize(PersonEntity) - const people = dataSource.getRepository(PersonEntity) + const db = new GitDb(engine) + await db.createTable(PersonTable) + const people = db.table(PersonTable) await people.insertMany(peopleRows(40)) const before = engine.getIndexStats() - // When: repository find emits simple equality SELECT statements. - const byPrimary = await people.find({ where: { id: "p7" } }) - const bySecondary = await people.find({ where: { team_id: "storage" } }) + // When: table select emits simple equality SELECT statements. + const byPrimary = await people.select({ id: "p7" }) + const bySecondary = await people.select({ team_id: "storage" }) // Then: both lookups use rebuilt equality indexes and match scan semantics. expect(byPrimary).toEqual([{ id: "p7", name: "Person 7", team_id: "runtime" }]) @@ -52,12 +56,12 @@ describe("index runtime", () => { it("rebuilds stale indexes on reopen", async () => { // Given: plaintext index artifacts are stale compared with the committed snapshot rows. const root = await mkdtemp(join(tmpdir(), "gitdb-index-reopen-")) - const first = await createGitDbDataSource({ - entities: [PersonEntity], + const first = await GitDb.open({ store: new LocalPlaintextStore({ root }), - synchronize: true, + syncSchema: true, + tables: [PersonTable], }) - await first.getRepository(PersonEntity).insertMany(peopleRows(12)) + await first.table(PersonTable).insertMany(peopleRows(12)) await writeFile( join(root, "gitdb/v1/people/indexes.json"), JSON.stringify({ columns: {}, rowCount: 0, version: 1 }), @@ -66,9 +70,9 @@ describe("index runtime", () => { // When: the database reopens and runs the same equality lookup. const reopened = await GitDbEngine.open({ store: new LocalPlaintextStore({ root }) }) - const dataSource = new GitDbDataSource(reopened) - const people = dataSource.getRepository(PersonEntity) - const storagePeople = await people.find({ where: { team_id: "storage" } }) + const db = new GitDb(reopened) + const people = db.table(PersonTable) + const storagePeople = await people.select({ team_id: "storage" }) // Then: stale index files are ignored and indexes rebuild from snapshot/log state. expect(storagePeople).toHaveLength(6) diff --git a/tests/no-postgres-facade.test.ts b/tests/no-postgres-facade.test.ts index dd85ccd..3d45493 100644 --- a/tests/no-postgres-facade.test.ts +++ b/tests/no-postgres-facade.test.ts @@ -138,32 +138,28 @@ describe("PostgreSQL facade removal", () => { } expect(scripts.example).toBe("corepack pnpm example:api") expect(scripts["example:api"]).toBe( - "corepack pnpm build && node examples/api-local/index.mjs && node examples/api-encrypted/index.mjs && node examples/api-encrypted-reopen/index.mjs", + "corepack pnpm build && node examples/api-plaintext/index.mjs && node examples/api-encrypted/index.mjs", ) - expect(scripts["example:api-local"]).toBe( - "corepack pnpm build && node examples/api-local/index.mjs", + expect(scripts["example:api-plaintext"]).toBe( + "corepack pnpm build && node examples/api-plaintext/index.mjs", ) expect(scripts["example:api-encrypted"]).toBe( "corepack pnpm build && node examples/api-encrypted/index.mjs", ) - expect(scripts["example:api-encrypted-reopen"]).toBe( - "corepack pnpm build && node examples/api-encrypted-reopen/index.mjs", - ) + expect(scripts).not.toHaveProperty("example:api-encrypted-reopen") + expect(scripts).not.toHaveProperty("example:api-local") expect(scripts).not.toHaveProperty("example:local-runtime") }) - it("removes the old wire-protocol service and generated ORM example files", async () => { + it("removes the old wire-protocol service and generated adapter example files", async () => { const removedPaths = [ "src/protocol", "src/http", "examples/express-prisma", "examples/local-runtime", + "examples/api-local", ] - const requiredPaths = [ - "examples/api-local/index.mjs", - "examples/api-encrypted/index.mjs", - "examples/api-encrypted-reopen/index.mjs", - ] + const requiredPaths = ["examples/api-plaintext/index.mjs", "examples/api-encrypted/index.mjs"] for (const path of removedPaths) { await expect(exists(path)).resolves.toBe(false) @@ -213,11 +209,12 @@ describe("PostgreSQL facade removal", () => { ]) }) - it("exports only the first-party runtime, storage engine, and repository API", async () => { + it("exports only the first-party runtime, table API, and storage engine", async () => { const index = await readFile("src/index.ts", "utf8") expect(index).not.toContain("createGitDbServer") expect(index).not.toContain("./protocol/") - expect(index).toContain("createGitDbDataSource") + expect(index).toContain("GitDb") + expect(index).toContain("defineTable") expect(index).toContain("GitDbEngine") }) }) diff --git a/tests/orm.test.ts b/tests/orm.test.ts deleted file mode 100644 index f71f85c..0000000 --- a/tests/orm.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -import { mkdtemp } from "node:fs/promises" -import { tmpdir } from "node:os" -import { join } from "node:path" -import { describe, expect, it } from "vitest" -import { createGitDbDataSource, defineEntity, type EntityDefinition } from "../src/orm/index.js" -import { LocalPlaintextStore } from "../src/storage/local-plaintext-store.js" -import { CountingBatchStore } from "./sql-engine-fixtures.js" - -type Person = { - readonly id: string - readonly name: string - readonly team_id: string -} - -type LoosePerson = { - readonly id: string - readonly name: string | { readonly nested: boolean } - readonly team_id: string -} - -const PersonEntity = defineEntity({ - columns: { - id: "STRING", - name: "STRING", - team_id: "STRING", - }, - primaryKey: "id", - tableName: "people", -}) - -const LoosePersonEntity = defineEntity({ - columns: { - id: "STRING", - name: "STRING", - team_id: "STRING", - }, - primaryKey: "id", - tableName: "people", -}) - -describe("GitDB ORM", () => { - it("rejects unsupported JSON column metadata", () => { - // Given: entity metadata that claims a JSON column. - const entity = { - columns: { - id: "STRING", - metadata: "JSON", - }, - primaryKey: "id", - tableName: "people", - } as unknown as EntityDefinition<{ readonly id: string; readonly metadata: string }> - - // When/Then: the ORM refuses a type it cannot persist faithfully yet. - expect(() => defineEntity(entity)).toThrow() - }) - - it("validates secondary index metadata", () => { - // Given: entity metadata with a declared equality-only secondary index. - const indexed = defineEntity({ - columns: { - id: "STRING", - name: "STRING", - team_id: "STRING", - }, - indexes: [{ columns: ["team_id"], name: "people_team_id_idx" }], - primaryKey: "id", - tableName: "people", - }) - - // When/Then: valid index metadata is preserved for the later index runtime. - expect(indexed.indexes).toEqual([{ columns: ["team_id"], name: "people_team_id_idx" }]) - expect(() => - defineEntity>({ - columns: { - id: "STRING", - name: "STRING", - team_id: "STRING", - }, - indexes: [{ columns: ["missing"], name: "people_missing_idx" }], - primaryKey: "id", - tableName: "people", - }), - ).toThrow("indexed column missing does not exist") - }) - - it("saves, finds, and deletes entities through a TypeORM-style repository", async () => { - // Given: a GitDB data source synchronized from entity metadata. - const root = await mkdtemp(join(tmpdir(), "gitdb-orm-")) - const dataSource = await createGitDbDataSource({ - entities: [PersonEntity], - store: new LocalPlaintextStore({ root }), - synchronize: true, - }) - const people = dataSource.getRepository(PersonEntity) - - // When: callers use repository methods instead of raw SQL. - await people.save({ id: "p1", name: "Lin", team_id: "storage" }) - await people.save({ id: "p2", name: "Ada", team_id: "runtime" }) - const found = await people.find({ where: { team_id: "storage" } }) - const one = await people.findOne({ where: { id: "p2" } }) - await people.delete({ id: "p1" }) - - // Then: the repository exposes typed CRUD ergonomics over the local runtime. - await expect(people.find()).resolves.toEqual([{ id: "p2", name: "Ada", team_id: "runtime" }]) - expect(found).toEqual([{ id: "p1", name: "Lin", team_id: "storage" }]) - expect(one).toEqual({ id: "p2", name: "Ada", team_id: "runtime" }) - }) - - it("inserts many entities in one repository transaction", async () => { - // Given: a synchronized repository backed by a counting v2 store. - const root = await mkdtemp(join(tmpdir(), "gitdb-orm-bulk-")) - const store = new CountingBatchStore(new LocalPlaintextStore({ root })) - const dataSource = await createGitDbDataSource({ - entities: [PersonEntity], - store, - synchronize: true, - }) - const people = dataSource.getRepository(PersonEntity) - store.resetCounts() - - // When: callers insert multiple rows through the repository bulk API. - const result = await people.insertMany([ - { id: "p1", name: "Lin", team_id: "storage" }, - { id: "p2", name: "Ada", team_id: "runtime" }, - ]) - - // Then: rows are visible and persistence used one engine batch boundary. - expect(result).toEqual({ inserted: 2 }) - await expect(people.find()).resolves.toEqual([ - { id: "p1", name: "Lin", team_id: "storage" }, - { id: "p2", name: "Ada", team_id: "runtime" }, - ]) - expect(store.commitBatchCalls).toBe(1) - expect(store.writeManifestCalls).toBeLessThanOrEqual(3) - }) - - it("treats empty and invalid bulk inserts as atomic repository operations", async () => { - // Given: a synchronized repository with no rows. - const root = await mkdtemp(join(tmpdir(), "gitdb-orm-bulk-invalid-")) - const store = new CountingBatchStore(new LocalPlaintextStore({ root })) - const dataSource = await createGitDbDataSource({ - entities: [PersonEntity], - store, - synchronize: true, - }) - const people = dataSource.getRepository(PersonEntity) - const loosePeople = dataSource.getRepository(LoosePersonEntity) - store.resetCounts() - - // When/Then: empty insertMany is a no-op and does not persist. - await expect(people.insertMany([])).resolves.toEqual({ inserted: 0 }) - expect(store.commitBatchCalls).toBe(0) - - // When/Then: invalid rows reject and leave no partial rows visible. - await expect( - loosePeople.insertMany([ - { id: "p1", name: "Lin", team_id: "storage" }, - { id: "bad", name: { nested: true }, team_id: "runtime" }, - ]), - ).rejects.toThrow("entity value for name") - await expect(people.find()).resolves.toEqual([]) - }) - - it("saves many entities with upsert semantics in one repository transaction", async () => { - // Given: existing rows and a counting store. - const root = await mkdtemp(join(tmpdir(), "gitdb-orm-save-many-")) - const store = new CountingBatchStore(new LocalPlaintextStore({ root })) - const dataSource = await createGitDbDataSource({ - entities: [PersonEntity], - store, - synchronize: true, - }) - const people = dataSource.getRepository(PersonEntity) - await people.insertMany([{ id: "p1", name: "Lin", team_id: "storage" }]) - const updates = Array.from({ length: 100 }, (_, index) => ({ - id: `p${index + 1}`, - name: `Person ${index + 1}`, - team_id: index % 2 === 0 ? "storage" : "runtime", - })) - store.resetCounts() - - // When: saveMany mixes overwrite and insert rows. - const result = await people.saveMany(updates) - - // Then: saveMany preserves save() overwrite semantics inside one batch. - const rows = await dataSource.query("SELECT * FROM people WHERE id = 'p1'") - expect(result).toEqual({ saved: 100 }) - expect(rows).toEqual([{ id: "p1", name: "Person 1", team_id: "storage" }]) - expect(store.commitBatchCalls).toBe(1) - expect(store.writeManifestCalls).toBeLessThanOrEqual(3) - }) -}) diff --git a/tests/package-metadata.test.ts b/tests/package-metadata.test.ts index b9889db..37e0fee 100644 --- a/tests/package-metadata.test.ts +++ b/tests/package-metadata.test.ts @@ -101,16 +101,17 @@ describe("npm publish metadata", () => { const index = await readFile("src/index.ts", "utf8") const migration = await readFile("docs/MIGRATION.md", "utf8") const publicSymbols = [ - "createGitDbDataSource", - "defineEntity", - "GitDbDataSource", - "GitDbRepository", - "GitDbEntityIndexDefinition", + "GitDb", + "openGitDb", + "defineTable", + "GitDbTable", + "GitDbTableIndexDefinition", "GitDbInsertResult", - "GitDbSaveManyResult", + "GitDbUpsertResult", "GitDbEngine", "LocalPlaintextStore", "LocalEncryptedStore", + "GitHubPlaintextStore", "GitHubEncryptedStore", "GitDbStore", "GitDbStoreV2", diff --git a/tests/site.test.ts b/tests/site.test.ts index 7f90062..abe4a1e 100644 --- a/tests/site.test.ts +++ b/tests/site.test.ts @@ -25,10 +25,10 @@ describe("website", () => { expect.arrayContaining([ "local-plaintext", "local-encrypted", - "orm-save-single", - "orm-insert-many", - "orm-save-many", - "orm-indexed-lookup", + "table-upsert-single", + "table-insert-many", + "table-upsert-many", + "table-indexed-lookup", "reopen", "compaction", ]), @@ -44,7 +44,7 @@ describe("website", () => { expect(workflow).toContain("path: site") expect(app).toContain("./benchmark.json") - expect(app).toContain("orm-indexed-lookup") + expect(app).toContain("table-indexed-lookup") expect(index).toContain("GitHub Repo + DB Runtime") expect(index).toContain("Guarantee Matrix") expect(index).toContain("Transactions") diff --git a/tsconfig.json b/tsconfig.json index a27ddd6..1b513ec 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,8 +12,6 @@ "isolatedModules": true, "esModuleInterop": true, "resolveJsonModule": true, - "experimentalDecorators": true, - "emitDecoratorMetadata": true, "target": "ES2022", "lib": ["ES2022"], "types": ["node"], From 35225f73658d2a772b59c99cd4acf0888a262a56 Mon Sep 17 00:00:00 2001 From: 3x-haust Date: Mon, 15 Jun 2026 11:38:33 +0900 Subject: [PATCH 2/2] =?UTF-8?q?ci:=20=ED=8E=98=EC=9D=B4=EC=A7=80=20?= =?UTF-8?q?=EC=9B=8C=ED=81=AC=ED=94=8C=EB=A1=9C=20=EC=83=9D=EC=84=B1=20?= =?UTF-8?q?=EC=84=A4=EC=A0=95=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pages.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index a049d02..6ec49ad 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -24,8 +24,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/configure-pages@v5 - with: - enablement: true - uses: actions/upload-pages-artifact@v3 with: path: site