From 4f081b5e2b08828c0efa7b8b7384a8ff6a5b0b9c Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 17 Aug 2026 20:29:16 -0500 Subject: [PATCH 1/8] docs(byodb): add BYODB pluggable database design spec --- .../specs/2026-08-17-byodb-design.md | 282 ++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-17-byodb-design.md diff --git a/docs/superpowers/specs/2026-08-17-byodb-design.md b/docs/superpowers/specs/2026-08-17-byodb-design.md new file mode 100644 index 0000000..265c343 --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-byodb-design.md @@ -0,0 +1,282 @@ +# BYODB — Bring Your Own Database + +**Date:** 2026-08-17 +**Status:** Approved for implementation +**Branch:** `feat/byodb` + +## Problem + +opencontext persists every saved context and bubble in a single JSON file at +`~/.opencontext/contexts.json`. `src/mcp/store.ts` reads and rewrites that entire +file on every operation. This works for a laptop with a few hundred entries and +fails everywhere else: + +- The whole store is rewritten per write, so cost grows with total size, not change size. +- Two processes writing concurrently (MCP server + HTTP server) can lose writes. +- There is no way to point opencontext at a database the user already runs. +- There is no remote option, so context cannot be shared across machines. + +## Goal + +Let users point opencontext at the database of their choice — embedded or remote — +without changing how the CLI, HTTP API, or MCP tools behave. The JSON file stays +the zero-configuration default so existing installs keep working untouched. + +## Scope + +**In scope:** contexts and bubbles — everything currently in `contexts.json`. + +**Out of scope:** `preferences.json`, `preferences.md`, and `memory.md` stay as +files on disk. They are generated artifacts that Claude reads directly from the +filesystem; moving them into a database would break that contract for no gain. + +**Explicitly not built (YAGNI):** connection pool tuning, vector or semantic +search, multi-user auth, schema migration beyond initial creation. + +## Architecture + +### The central change: the store becomes async + +Today `createStore()` returns an object of synchronous methods. Every database +driver is asynchronous, so the store interface must become async. Both consumers +(`src/server.ts` Express handlers and `src/mcp/server.ts` tool handlers) already +execute inside async contexts, so this ripple is mechanical: add `async`/`await`. + +### Approach: SQL-generic core plus dialect bindings + +SQLite, Postgres, and DuckDB are all SQL databases. Rather than write the same +CRUD five times, the CRUD is written **once** against a small `SqlDriver` +interface, and each engine supplies a `Dialect` carrying its parameter-placeholder +style and its DDL. JSON and SurrealDB get bespoke adapters because neither is SQL. + +This is roughly 40% of the code of five independent adapters, and — more +importantly — search and ordering semantics cannot drift between SQL backends +because there is only one implementation of them. + +### Layout + +``` +src/store/ +├── types.ts ContextStoreAdapter, AdapterInfo, DbScheme +├── dsn.ts parse / validate / redact connection strings +├── config.ts ~/.opencontext/config.json read + write (mode 0600) +├── resolve.ts precedence chain producing the effective DSN +├── registry.ts scheme → adapter loader, driver-availability probing +├── index.ts createStore(dsn) factory +├── manager.ts StoreManager — lazy connect, hot reconnect +├── migrate.ts copy all data between two adapters +├── adapters/ +│ ├── json.ts today's file store, async (DEFAULT) +│ ├── sql.ts shared SQL CRUD over SqlDriver + Dialect +│ └── surreal.ts bespoke, SurrealQL +└── drivers/ + ├── sqlite.ts node:sqlite built-in; @libsql/client for libsql:// + ├── postgres.ts pg + └── duckdb.ts @duckdb/node-api +``` + +`src/mcp/store.ts` becomes a thin re-export of the new module so any external +importer keeps resolving. + +### The adapter interface + +```ts +interface ContextStoreAdapter { + readonly info: AdapterInfo; + connect(): Promise; + close(): Promise; + ping(): Promise; + + saveContext(content, tags?, source?, bubbleId?): Promise; + recallContext(query): Promise; + listContexts(tag?): Promise; + listContextsByBubble(bubbleId): Promise; + getContext(id): Promise; + updateContext(id, content, tags?, bubbleId?): Promise; + deleteContext(id): Promise; + searchContexts(query): Promise; + + createBubble(name, description?): Promise; + listBubbles(): Promise; + getBubble(id): Promise; + updateBubble(id, name, description?): Promise; + deleteBubble(id, deleteContexts?): Promise; +} +``` + +Method signatures are otherwise unchanged from today's store, so call sites only +gain an `await`. + +`AdapterInfo` is `{ scheme, label, target, remote }` where `target` is always +**redacted** — it is returned over HTTP to the UI. + +## Connection strings + +| Scheme | Driver | Dependency | +|---|---|---| +| `json:///path/contexts.json` | `node:fs` | none — **default** | +| `sqlite:///path/oc.db`, `sqlite::memory:` | `node:sqlite` | **none** (Node 25 built-in) | +| `libsql://host?authToken=…` | `@libsql/client` | optional | +| `postgres://user:pass@host:5432/db` | `pg` | optional | +| `duckdb:///path/oc.duckdb` | `@duckdb/node-api` | optional | +| `surrealdb://user:pass@host:8000/ns/db` | `surrealdb` | optional | + +`postgresql://` is accepted as an alias for `postgres://`. SurrealDB accepts +`ws://` and `wss://` aliases; its path segment is `//`. + +### Resolution precedence + +1. `OPENCONTEXT_DB_URL` environment variable +2. `database.url` in `~/.opencontext/config.json` +3. `OPENCONTEXT_STORE_PATH` — legacy, mapped to `json://` +4. Default: `json://~/.opencontext/contexts.json` + +Environment always wins over the config file, so a container can override whatever +a user saved from the UI. + +### Optional drivers + +Drivers are declared as `peerDependencies` with `peerDependenciesMeta.optional`, +which npm does **not** auto-install. The default install and the Docker image stay +lean. An adapter `await import()`s its driver on first connect; a missing module +produces an actionable error rather than a stack trace: + +``` +Postgres driver is not installed. +Install it with: npm install pg +``` + +## SQL schema + +```sql +CREATE TABLE IF NOT EXISTS oc_bubbles ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS oc_contexts ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + tags TEXT NOT NULL, -- JSON-encoded string[] + source TEXT NOT NULL, + bubble_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +``` + +`tags` is a JSON-encoded text column rather than a native array or JSONB. Every +target engine stores and compares text identically, which keeps the shared CRUD +free of dialect branches. Tag filtering and search run over the encoded text with +`LOWER(...) LIKE ...`, preserving today's case-insensitive substring semantics +exactly. + +Timestamps are ISO-8601 strings, matching the existing JSON store, so migration is +a straight copy with no conversion. + +### Ordering contract + +Every adapter returns contexts and bubbles ordered by **`created_at` ascending, +then `id` ascending**. + +This is a deliberate, documented change from the JSON store's implicit insertion +order. Two entries written in the same millisecond previously came back in +insertion order; they now come back in UUID order. In exchange, ordering is +*identical and deterministic* across all five backends, which makes a shared +conformance suite possible. The JSON adapter sorts on read to match. + +## Configuration surface + +### HTTP API + +| Route | Purpose | +|---|---| +| `GET /api/db/status` | current adapter, redacted target, connected, entry/bubble counts | +| `GET /api/db/adapters` | supported schemes, labels, and whether each driver is installed | +| `POST /api/db/test` | connect to a candidate DSN, run `ping()`, disconnect; report ok or the driver error | +| `PUT /api/db/config` | persist DSN to config file, hot-swap the live store | +| `POST /api/db/migrate` | copy all contexts and bubbles from the live store into a target DSN | + +`POST /api/db/migrate` takes `{ url, mode }` where `mode` is `"copy"` (default, +additive) or `"replace"` (target is cleared first). It returns counts of what was +transferred. + +### Web UI + +New route `/settings` rendering `DatabaseSettings.tsx`, linked from the sidebar: + +- current backend, with a badge for local vs remote +- adapter picker that fills in a scheme-appropriate DSN template +- connection-string field with a **Test connection** button reporting the real driver error on failure +- **Save** to persist, and **Migrate my data here** to copy the existing store across +- an install hint when the chosen adapter's driver is not present + +### CLI + +``` +opencontext db status +opencontext db adapters +opencontext db test +opencontext db migrate --to [--replace] +``` + +## Store lifecycle + +`StoreManager` owns the live adapter. It connects **lazily on first use** rather +than at module load, which keeps `src/server.ts` synchronously importable — the +test suite imports `app` directly via supertest, and top-level `await` there would +change module semantics for every existing test. + +`reconnect(dsn)` closes the current adapter, opens the new one, and swaps it in. +If the new adapter fails to connect, the previous one is retained and the error is +returned — a bad connection string entered in the UI cannot take the store down. + +## Error handling + +- Unknown or malformed DSN → `400` from the API, non-zero exit from the CLI, with the list of supported schemes. +- Missing optional driver → error naming the exact `npm install` command. +- Connection failure → the driver's own message is surfaced verbatim (minus credentials) because it is the only useful diagnostic. +- Migration failure → partial progress is reported; the source store is never mutated by a migration. + +## Security + +Connection strings carry passwords. + +- `~/.opencontext/config.json` is written with mode `0600`. +- `redactDsn()` replaces the password component with `***`; every API response, log line, and `AdapterInfo.target` passes through it. +- The UI never receives a stored password back — the DSN field shows the redacted form and only sends a new value when the user types a full replacement. +- Nothing leaves the machine. Remote connections go directly from the user's process to the user's database, consistent with the project's local-only privacy stance. + +## Testing + +The core testing move is a **shared conformance suite** at +`tests/store/conformance.ts`, exported as a function taking an adapter factory. It +covers every case in today's `tests/mcp/store.test.ts` plus bubble unassign-vs-cascade +delete, tag filtering, multi-term search, and the ordering contract. + +It runs unconditionally against: + +- **JSON** — no external dependency +- **SQLite** — `node:sqlite` is built into Node 25, so still no external dependency + +and against Postgres, DuckDB, and SurrealDB only when their connection +environment variables are set, so CI stays green without containers and a +developer with a local Postgres gets real coverage for free. + +Additional tests: + +- `tests/store/dsn.test.ts` — parsing for every scheme, aliases, malformed input, and redaction +- `tests/store/config.test.ts` — precedence chain and file permissions +- `tests/store/migrate.test.ts` — JSON → SQLite copy and replace, verified by reading back through the target adapter +- `tests/server.test.ts` — updated mock store returns promises; new `/api/db/*` route tests +- `ui/src/components/__tests__/DatabaseSettings.test.tsx` — render, test-connection success and failure, save + +## Rollout + +No migration is required. An existing install with no configuration continues to +resolve to `json://~/.opencontext/contexts.json` and reads the same file it always +has. Users opt in by setting `OPENCONTEXT_DB_URL` or saving a connection from the +settings page, then running a migration to carry their history across. From 41996522b18d4d4fc9efba1e1c0f0815c7ae3cfb Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 17 Aug 2026 20:37:13 -0500 Subject: [PATCH 2/8] fix(ui): unblock Pages build by allowing eslint 10 in react-hooks peer range Dependabot bumped eslint to ^10.8.1 in ui/ while leaving eslint-plugin-react-hooks at ^7.0.1, whose peer range stops at eslint 9. npm ci then fails with ERESOLVE, so the Cloudflare Pages build never reaches the build step. eslint-plugin-react-hooks 7.1.1 widens its peer range to include ^10.0.0. typescript-eslint 8.67.0 and eslint-plugin-react-refresh 0.4.24 already accept eslint 10, so this is the only change needed. Verified against origin/main: npm ci, npm run build, and the root package build all succeed. --- ui/package-lock.json | 10 +++++----- ui/package.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index c878267..a9f5769 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -30,7 +30,7 @@ "@vitejs/plugin-react": "^5.2.0", "@vitest/coverage-v8": "^4.1.8", "eslint": "^10.8.1", - "eslint-plugin-react-hooks": "^7.0.1", + "eslint-plugin-react-hooks": "^7.1.1", "eslint-plugin-react-refresh": "^0.4.24", "globals": "^16.5.0", "jsdom": "^26.1.0", @@ -5371,9 +5371,9 @@ } }, "node_modules/eslint-plugin-react-hooks": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.0.1.tgz", - "integrity": "sha512-O0d0m04evaNzEPoSW+59Mezf8Qt0InfgGIBJnpC0h3NH/WjUAR7BIKUfysC6todmtiZ/A0oUVS8Gce0WhBrHsA==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.1.1.tgz", + "integrity": "sha512-f2I7Gw6JbvCexzIInuSbZpfdQ44D7iqdWX01FKLvrPgqxoE7oMj8clOfto8U6vYiz4yd5oKu39rRSVOe1zRu0g==", "dev": true, "license": "MIT", "dependencies": { @@ -5387,7 +5387,7 @@ "node": ">=18" }, "peerDependencies": { - "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0" + "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 || ^10.0.0" } }, "node_modules/eslint-plugin-react-refresh": { diff --git a/ui/package.json b/ui/package.json index 88b5a04..5700955 100644 --- a/ui/package.json +++ b/ui/package.json @@ -38,7 +38,7 @@ "@vitejs/plugin-react": "^5.2.0", "@vitest/coverage-v8": "^4.1.8", "eslint": "^10.8.1", - "eslint-plugin-react-hooks": "^7.0.1", + "eslint-plugin-react-hooks": "^7.1.1", "eslint-plugin-react-refresh": "^0.4.24", "globals": "^16.5.0", "jsdom": "^26.1.0", From ae50f51a6ef05b186cc7e104afc563004720a73c Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 17 Aug 2026 21:08:12 -0500 Subject: [PATCH 3/8] =?UTF-8?q?feat(store):=20BYODB=20=E2=80=94=20pluggabl?= =?UTF-8?q?e=20database=20backends?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the single-JSON-file context store with an async adapter interface backed by 15 databases. The JSON file remains the zero-config default, so existing installs are unaffected: with no configuration opencontext reads the same file it always has, and OPENCONTEXT_STORE_PATH still works. Backends: JSON, in-memory, SQLite, DuckDB, libSQL/Turso, Cloudflare D1, PostgreSQL, Google Cloud SQL, MySQL/MariaDB, SQL Server/Azure SQL, MongoDB, Redis/Valkey, Google Firestore, Amazon DynamoDB, SurrealDB. CRUD is implemented once per family rather than once per backend: adapters/sql.ts one implementation for all 8 SQL engines; each supplies a Dialect (placeholder style, DDL, concat) adapters/document.ts one implementation for all document/KV stores behind a six-method DocumentDriver adapters/json.ts the file store adapters/surreal.ts bespoke This keeps search and ordering semantics from drifting between backends, since there is only one implementation of them. Drivers are optional peer dependencies loaded via dynamic import, so npm does not install them and the package builds with none present. A missing driver produces an install instruction rather than a module-resolution error. The store interface is now async, since every driver is. Signatures are otherwise unchanged, so the HTTP and MCP servers only gained awaits. src/mcp/store.ts is a deprecated re-export. Adds: - tests/store/conformance.ts — ~50 tests defining the storage contract, run against every adapter; JSON, SQLite and in-memory run with no external service so all three shared cores are covered in ordinary CI - docker-compose.test.yml — the rest of the backends for local UAT - /api/db/{status,adapters,test,config,migrate} and a Database page in the UI - opencontext db {status,adapters,test,use,reset,migrate} - DB_DRIVERS build arg so a Docker image can bake in the drivers it needs Connection strings carry passwords, so ~/.opencontext/config.json is written 0600 and credentials are redacted from every response, log line and UI field. Drivers receive dsn.canonical rather than dsn.raw, which preserves rediss:// and mongodb+srv:// — collapsing the first would silently disable TLS. --- CLAUDE.md | 67 ++- Dockerfile | 14 +- README.md | 103 +++- docker-compose.test.yml | 75 +++ docker-compose.yml | 8 + .../specs/2026-08-17-byodb-design.md | 45 +- package-lock.json | 52 ++ package.json | 64 ++- src/index.ts | 117 ++++ src/mcp/index.ts | 6 +- src/mcp/server.ts | 55 +- src/mcp/store.ts | 239 +------- src/server.ts | 228 ++++++-- src/store/adapters/document.ts | 238 ++++++++ src/store/adapters/json.ts | 237 ++++++++ src/store/adapters/sql.ts | 530 ++++++++++++++++++ src/store/adapters/surreal.ts | 401 +++++++++++++ src/store/config.ts | 104 ++++ src/store/drivers/d1.ts | 63 +++ src/store/drivers/duckdb.ts | 74 +++ src/store/drivers/dynamodb.ts | 212 +++++++ src/store/drivers/firestore.ts | 78 +++ src/store/drivers/memory.ts | 60 ++ src/store/drivers/mongodb.ts | 113 ++++ src/store/drivers/mssql.ts | 107 ++++ src/store/drivers/mysql.ts | 97 ++++ src/store/drivers/optional.ts | 33 ++ src/store/drivers/postgres.ts | 121 ++++ src/store/drivers/redis.ts | 216 +++++++ src/store/drivers/sqlite.ts | 92 +++ src/store/dsn.ts | 402 +++++++++++++ src/store/index.ts | 180 ++++++ src/store/manager.ts | 80 +++ src/store/migrate.ts | 68 +++ src/store/types.ts | 95 ++++ tests/mcp/store.test.ts | 307 ---------- tests/server.test.ts | 22 +- tests/store/backends.test.ts | 116 ++++ tests/store/config.test.ts | 116 ++++ tests/store/conformance.ts | 392 +++++++++++++ tests/store/dsn.test.ts | 301 ++++++++++ tests/store/json-adapter.test.ts | 79 +++ tests/store/migrate.test.ts | 109 ++++ tests/store/sqlite-adapter.test.ts | 34 ++ ui/src/App.tsx | 2 + ui/src/components/DatabaseSettings.tsx | 353 ++++++++++++ ui/src/components/Layout.tsx | 3 +- .../__tests__/DatabaseSettings.test.tsx | 207 +++++++ 48 files changed, 6091 insertions(+), 624 deletions(-) create mode 100644 docker-compose.test.yml create mode 100644 src/store/adapters/document.ts create mode 100644 src/store/adapters/json.ts create mode 100644 src/store/adapters/sql.ts create mode 100644 src/store/adapters/surreal.ts create mode 100644 src/store/config.ts create mode 100644 src/store/drivers/d1.ts create mode 100644 src/store/drivers/duckdb.ts create mode 100644 src/store/drivers/dynamodb.ts create mode 100644 src/store/drivers/firestore.ts create mode 100644 src/store/drivers/memory.ts create mode 100644 src/store/drivers/mongodb.ts create mode 100644 src/store/drivers/mssql.ts create mode 100644 src/store/drivers/mysql.ts create mode 100644 src/store/drivers/optional.ts create mode 100644 src/store/drivers/postgres.ts create mode 100644 src/store/drivers/redis.ts create mode 100644 src/store/drivers/sqlite.ts create mode 100644 src/store/dsn.ts create mode 100644 src/store/index.ts create mode 100644 src/store/manager.ts create mode 100644 src/store/migrate.ts create mode 100644 src/store/types.ts delete mode 100644 tests/mcp/store.test.ts create mode 100644 tests/store/backends.test.ts create mode 100644 tests/store/config.test.ts create mode 100644 tests/store/conformance.ts create mode 100644 tests/store/dsn.test.ts create mode 100644 tests/store/json-adapter.test.ts create mode 100644 tests/store/migrate.test.ts create mode 100644 tests/store/sqlite-adapter.test.ts create mode 100644 ui/src/components/DatabaseSettings.tsx create mode 100644 ui/src/components/__tests__/DatabaseSettings.test.tsx diff --git a/CLAUDE.md b/CLAUDE.md index f2550aa..a184491 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -139,7 +139,9 @@ The MCP server (`src/mcp/`) provides 6 tools: | `update_context` | Update content/tags of an existing context | | `delete_context` | Remove a context by ID | -**Storage**: `~/.opencontext/contexts.json` (override with `OPENCONTEXT_STORE_PATH` env var) +**Storage**: pluggable (BYODB). Defaults to `~/.opencontext/contexts.json`; set +`OPENCONTEXT_DB_URL` to use any of the 15 supported backends. `OPENCONTEXT_STORE_PATH` still +works and maps onto the JSON adapter. **Running**: ```bash @@ -177,6 +179,66 @@ node dist/mcp/index.js --- +## BYODB — Pluggable Databases + +The context store is an **async adapter interface** (`src/store/types.ts`), not a JSON file. +Fifteen backends implement it. When touching this area, the important things to know: + +### Three adapter families + +| Family | Shared implementation | Backends | +|---|---|---| +| File | `adapters/json.ts` | json | +| SQL | `adapters/sql.ts` + a `Dialect` per engine | sqlite, libsql, d1, duckdb, postgres, cloudsql, mysql, mssql | +| Document / KV | `adapters/document.ts` + a `DocumentDriver` per engine | memory, mongodb, redis, firestore, dynamodb | +| Multi-model | `adapters/surreal.ts` (bespoke) | surrealdb | + +**CRUD is written once per family.** Adding a SQL engine means supplying a `Dialect` +(placeholder style, DDL, concat) and a ~50-line driver. Adding a NoSQL engine means +implementing six `DocumentDriver` methods. Do not reimplement the storage contract per backend. + +### The conformance suite is the contract + +`tests/store/conformance.ts` holds ~50 tests that every adapter must pass. It is the only +definition of correct behaviour, and **it must never be weakened to make a backend pass** — +if a backend cannot satisfy it, that is a finding to document, not a test to relax. + +It runs with no external services against **json**, **sqlite** and **memory** (covering all +three families), and against the rest when their connection strings are in the environment: + +```bash +docker compose -f docker-compose.test.yml up -d +OPENCONTEXT_TEST_POSTGRES_URL="postgres://opencontext:opencontext@127.0.0.1:55432/opencontext" \ + npm run test:backends +docker compose -f docker-compose.test.yml down -v +``` + +### Invariants to preserve + +- **Ordering**: every list method returns `createdAt ASC, id ASC`. Identical across backends. +- **Search semantics**: case-insensitive substring over content/tags/source; `searchContexts` + requires all terms. +- **Absent vs null**: an unset `bubbleId`/`description` must come back `undefined`, never + `null`. Several drivers need explicit configuration to honour this. +- **Read-your-writes**: a write must be visible to the next read. DynamoDB needs + `ConsistentRead: true` for this. +- **Credentials never leak**: everything user-visible goes through `redactDsn()`. +- **Drivers pass `dsn.canonical`, never `dsn.raw`** — client libraries reject the aliases we + advertise, and `rediss://` / `mongodb+srv://` carry meaning that must survive. + +### Adding a backend + +1. Add the scheme to `DbScheme` (`types.ts`) and `SUPPORTED_SCHEMES` (`dsn.ts`) +2. Parse any backend-specific fields in `dsn.ts` +3. Write a driver in `src/store/drivers/` — a `Dialect` + `SqlDriver`, or a `DocumentDriver` +4. Register it in `ADAPTERS` and the `build()` switch in `src/store/index.ts` +5. Load it with `importOptional()` so a missing package produces an install instruction +6. Add it to `peerDependencies` + `peerDependenciesMeta` as optional +7. Add a service to `docker-compose.test.yml` and a line to `tests/store/backends.test.ts` +8. Run the conformance suite against it until all tests pass + +--- + ## Key Concepts ### Conversion Pipeline @@ -286,6 +348,9 @@ docker push adityakarnam/opencontext:latest | `npm run build` | Compile TypeScript (CLI + server + MCP) | | `npm run server` | Run HTTP server in dev mode | | `npm run mcp:server` | Run MCP server in dev mode | +| `npm run test:backends` | Run store conformance against configured databases | +| `npm run db -- status` | Show the current database backend | +| `npm run db -- adapters` | List every backend and whether its driver is installed | | `cd ui && npm run dev` | Start UI dev server | | `cd ui && npm run build` | Build UI | diff --git a/Dockerfile b/Dockerfile index 41e18b5..18fe506 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,9 +33,19 @@ FROM node:25-slim WORKDIR /app -# Install production dependencies only +# Database drivers to bake in (BYODB). Empty by default, which keeps the image +# small — opencontext runs on the built-in JSON, memory, SQLite and Cloudflare D1 +# backends with no driver at all. +# +# docker build --build-arg DB_DRIVERS="pg" -t opencontext . +# docker build --build-arg DB_DRIVERS="mongodb redis" -t opencontext . +ARG DB_DRIVERS="" + +# Install production dependencies only. Optional peer dependencies are not +# auto-installed by npm, so only the drivers named above are added. COPY package.json package-lock.json* ./ -RUN npm ci --omit=dev +RUN npm ci --omit=dev \ + && if [ -n "$DB_DRIVERS" ]; then npm install --no-save $DB_DRIVERS; fi # Copy compiled server + MCP + CLI COPY --from=server-builder /app/dist ./dist diff --git a/README.md b/README.md index c969129..f8bdc95 100644 --- a/README.md +++ b/README.md @@ -90,12 +90,113 @@ Switching AI assistants means losing all prior context — your communication st - Persistent context across Claude chats - Save, recall, search, and tag memories - Works with Claude Code & Claude Desktop -- Local JSON store at `~/.opencontext/` +- Store it in **any database you like** (BYODB) + +--- + +## 🗄️ Bring Your Own Database (BYODB) + +By default opencontext keeps everything in a JSON file at `~/.opencontext/contexts.json` — zero +configuration, nothing to install. When you outgrow that, point it at any of **15 backends** +without changing how the CLI, the web UI, or the MCP tools behave. + +```bash +# See what is available and what is installed +opencontext db adapters + +# Try a connection without committing to it +opencontext db test "postgres://user:pass@localhost:5432/opencontext" + +# Switch to it, and bring your existing history along +opencontext db use "postgres://user:pass@localhost:5432/opencontext" +opencontext db migrate --to "postgres://user:pass@localhost:5432/opencontext" +``` + +Or use the **Database** page in the web UI: pick a backend, test the connection, save it, and +copy your data across — no terminal required. + +### Supported backends + +| Backend | Connection string | Install | +|---|---|---| +| **JSON file** *(default)* | `json:///path/to/contexts.json` | — built in | +| **In-memory** | `memory://` | — built in | +| **SQLite** | `sqlite:///path/to/opencontext.db` | — built in (`node:sqlite`) | +| **Cloudflare D1** | `d1://ACCOUNT_ID/DATABASE_ID?apiToken=TOKEN` | — built in (HTTP) | +| **DuckDB** | `duckdb:///path/to/opencontext.duckdb` | `npm i @duckdb/node-api` | +| **libSQL / Turso** | `libsql://DB.turso.io?authToken=TOKEN` | `npm i @libsql/client` | +| **PostgreSQL** | `postgres://user:pass@host:5432/db` | `npm i pg` | +| **Google Cloud SQL** | `cloudsql://user:pass@PROJECT:REGION:INSTANCE/db` | `npm i @google-cloud/cloud-sql-connector pg` | +| **MySQL / MariaDB** | `mysql://user:pass@host:3306/db` | `npm i mysql2` | +| **SQL Server / Azure SQL** | `mssql://user:pass@host:1433/db` | `npm i mssql` | +| **MongoDB** | `mongodb://user:pass@host:27017/db` | `npm i mongodb` | +| **Redis / Valkey** | `redis://host:6379` | `npm i redis` | +| **Google Firestore** | `firestore://PROJECT_ID` | `npm i @google-cloud/firestore` | +| **Amazon DynamoDB** | `dynamodb://REGION/TABLE` | `npm i @aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb` | +| **SurrealDB** | `surrealdb://user:pass@host:8000/ns/db` | `npm i surrealdb` | + +Drivers are **optional peer dependencies** — nothing is installed until you ask for a backend +that needs it, so the default install and the Docker image stay small. Pick a backend whose +driver is missing and opencontext tells you exactly what to run. + +### Managed services + +Most managed databases speak a protocol already listed above, so they need no special support: + +| Service | Use | +|---|---| +| Neon, Supabase, Amazon RDS/Aurora, Azure Database for PostgreSQL, CockroachDB, Timescale | `postgres://…` (add `?sslmode=require`) | +| PlanetScale, Azure Database for MySQL, Cloud SQL for MySQL, Aurora MySQL | `mysql://…?ssl=true` | +| Azure Cosmos DB (MongoDB API), MongoDB Atlas | `mongodb://…` / `mongodb+srv://…` | +| Upstash, ElastiCache, Valkey | `redis://…` or `rediss://…` for TLS | +| Turso | `libsql://…` | + +### Configuration + +The store is resolved in this order — the first one that is set wins: + +1. `OPENCONTEXT_DB_URL` environment variable +2. `database.url` in `~/.opencontext/config.json` (what the UI and `db use` write) +3. `OPENCONTEXT_STORE_PATH` — the legacy setting, still honoured +4. Default: `~/.opencontext/contexts.json` + +Because the environment wins, a container can pin the database regardless of what is saved +locally. **Existing installs need to do nothing** — with no configuration at all, opencontext +reads the same JSON file it always has. + +```bash +# Docker with Postgres +docker run -p 3000:3000 \ + -e OPENCONTEXT_DB_URL="postgres://user:pass@db.internal:5432/opencontext" \ + adityakarnam/opencontext:latest +``` + +### Choosing a backend + +- **Staying on one machine?** The default JSON file is fine. Move to **SQLite** when you have + thousands of contexts or run the HTTP and MCP servers at once — it needs no install and + writes only what changed instead of rewriting the whole store. +- **Sharing context across machines?** Any of the remote backends. **PostgreSQL** is the + best-supported, and every predicate runs in the database. +- **Already run a database?** Use it. That is the point. + +A note on how search behaves: the SQL backends push filtering down into the database. The +document and key-value backends (MongoDB, Redis, Firestore, DynamoDB) have no portable +case-insensitive substring predicate, so opencontext reads the context collection and filters +in memory. Results are identical — every backend passes the same conformance suite — but on a +very large store a SQL backend will be faster. + +### Security + +Connection strings carry passwords, so `~/.opencontext/config.json` is written with owner-only +(`0600`) permissions and credentials are redacted from every API response, log line, and UI +field. Nothing is sent anywhere: your process connects directly to your database. + --- ## 🚀 Quick Start diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..6a0ea5b --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,75 @@ +# Databases for BYODB conformance testing. +# +# docker compose -f docker-compose.test.yml up -d +# npm run test:backends +# docker compose -f docker-compose.test.yml down -v +# +# Every service binds a non-default host port so it cannot collide with a real +# database already running on the machine. + +services: + postgres: + image: postgres:17-alpine + environment: + POSTGRES_USER: opencontext + POSTGRES_PASSWORD: opencontext + POSTGRES_DB: opencontext + ports: + - '55432:5432' + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U opencontext'] + interval: 3s + retries: 20 + + mysql: + image: mysql:8.4 + environment: + MYSQL_ROOT_PASSWORD: opencontext + MYSQL_DATABASE: opencontext + MYSQL_USER: opencontext + MYSQL_PASSWORD: opencontext + ports: + - '53306:3306' + healthcheck: + test: ['CMD', 'mysqladmin', 'ping', '-h', '127.0.0.1', '-popencontext'] + interval: 5s + retries: 30 + + mssql: + image: mcr.microsoft.com/mssql/server:2022-latest + environment: + ACCEPT_EULA: 'Y' + MSSQL_SA_PASSWORD: 'OpenContext!2026' + MSSQL_PID: Developer + ports: + - '51433:1433' + + mongodb: + image: mongo:8 + ports: + - '57017:27017' + healthcheck: + test: ['CMD', 'mongosh', '--quiet', '--eval', 'db.runCommand({ping:1})'] + interval: 3s + retries: 20 + + redis: + image: redis:7-alpine + ports: + - '56379:6379' + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 3s + retries: 20 + + surrealdb: + image: surrealdb/surrealdb:latest + command: start --user root --pass root --bind 0.0.0.0:8000 memory + ports: + - '58000:8000' + + dynamodb: + image: amazon/dynamodb-local:latest + command: -jar DynamoDBLocal.jar -inMemory -sharedDb + ports: + - '58001:8000' diff --git a/docker-compose.yml b/docker-compose.yml index e8d56b8..4c9fec3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,10 @@ services: - opencontext-data:/root/.opencontext environment: - OPENCONTEXT_STORE_PATH=/root/.opencontext/contexts.json + # BYODB — point at any supported database instead of the JSON file. + # The driver must be baked in at build time: + # docker compose build --build-arg DB_DRIVERS="pg" app + # - OPENCONTEXT_DB_URL=postgres://user:pass@db:5432/opencontext # Ollama runs on the host — host.docker.internal resolves to the host machine IP - OLLAMA_HOST=http://host.docker.internal:11434 - PORT=3000 @@ -49,6 +53,10 @@ services: - opencontext-data:/root/.opencontext environment: - OPENCONTEXT_STORE_PATH=/root/.opencontext/contexts.json + # BYODB — point at any supported database instead of the JSON file. + # The driver must be baked in at build time: + # docker compose build --build-arg DB_DRIVERS="pg" app + # - OPENCONTEXT_DB_URL=postgres://user:pass@db:5432/opencontext command: ["node", "dist/mcp/index.js"] volumes: diff --git a/docs/superpowers/specs/2026-08-17-byodb-design.md b/docs/superpowers/specs/2026-08-17-byodb-design.md index 265c343..8734fad 100644 --- a/docs/superpowers/specs/2026-08-17-byodb-design.md +++ b/docs/superpowers/specs/2026-08-17-byodb-design.md @@ -1,7 +1,10 @@ # BYODB — Bring Your Own Database **Date:** 2026-08-17 -**Status:** Approved for implementation +**Status:** Implemented +**Revised:** 2026-08-17 — scope widened from 5 backends to 15 during implementation +(Azure SQL, Cloud SQL, DynamoDB, MySQL, MongoDB, Redis, Firestore, D1, memory added +at the user's request). **Branch:** `feat/byodb` ## Problem @@ -42,16 +45,25 @@ driver is asynchronous, so the store interface must become async. Both consumers (`src/server.ts` Express handlers and `src/mcp/server.ts` tool handlers) already execute inside async contexts, so this ripple is mechanical: add `async`/`await`. -### Approach: SQL-generic core plus dialect bindings +### Approach: shared cores plus thin per-engine bindings -SQLite, Postgres, and DuckDB are all SQL databases. Rather than write the same -CRUD five times, the CRUD is written **once** against a small `SqlDriver` -interface, and each engine supplies a `Dialect` carrying its parameter-placeholder -style and its DDL. JSON and SurrealDB get bespoke adapters because neither is SQL. +The backends fall into families, and CRUD is written **once per family**: -This is roughly 40% of the code of five independent adapters, and — more -importantly — search and ordering semantics cannot drift between SQL backends -because there is only one implementation of them. +| Family | Shared implementation | Per-engine work | Backends | +|---|---|---|---| +| File | `adapters/json.ts` | — | json | +| SQL | `adapters/sql.ts` + `Dialect` | placeholder style, DDL, concat, ~50-line driver | sqlite, libsql, d1, duckdb, postgres, cloudsql, mysql, mssql | +| Document / KV | `adapters/document.ts` + `DocumentDriver` | six methods (get/put/remove/list/ping/connect) | memory, mongodb, redis, firestore, dynamodb | +| Multi-model | `adapters/surreal.ts` | bespoke | surrealdb | + +This is a small fraction of the code of fifteen independent adapters, and — more +importantly — search and ordering semantics cannot drift between backends in a +family, because there is only one implementation of them. + +The `Dialect` carries everything that genuinely differs between SQL engines: +placeholder style (`?` / `$1` / `@p1`), DDL (SQL Server has no +`CREATE TABLE IF NOT EXISTS`; MySQL cannot index an unbounded `TEXT` key), and +string concatenation (`||` versus `+` versus `CONCAT()`). ### Layout @@ -257,14 +269,17 @@ The core testing move is a **shared conformance suite** at covers every case in today's `tests/mcp/store.test.ts` plus bubble unassign-vs-cascade delete, tag filtering, multi-term search, and the ordering contract. -It runs unconditionally against: +It runs unconditionally against one backend from each family, so all three shared +implementations are covered with no external dependency: -- **JSON** — no external dependency -- **SQLite** — `node:sqlite` is built into Node 25, so still no external dependency +- **JSON** — the file adapter +- **SQLite** — the shared SQL core, via Node's built-in `node:sqlite` +- **memory** — the shared document core -and against Postgres, DuckDB, and SurrealDB only when their connection -environment variables are set, so CI stays green without containers and a -developer with a local Postgres gets real coverage for free. +Every other backend runs when its connection string is in the environment. +`docker-compose.test.yml` brings up Postgres, MySQL, SQL Server, MongoDB, Redis, +SurrealDB and DynamoDB Local on non-default ports so they cannot collide with +anything already running. Additional tests: diff --git a/package-lock.json b/package-lock.json index 56ed689..c61b835 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,58 @@ }, "engines": { "node": ">=25.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.0.0", + "@aws-sdk/lib-dynamodb": "^3.0.0", + "@duckdb/node-api": "^1.0.0", + "@google-cloud/cloud-sql-connector": "^1.0.0", + "@google-cloud/firestore": "^7.0.0", + "@libsql/client": "^0.15.0", + "mongodb": "^6.0.0", + "mssql": "^11.0.0", + "mysql2": "^3.0.0", + "pg": "^8.0.0", + "redis": "^5.0.0", + "surrealdb": "^2.0.0" + }, + "peerDependenciesMeta": { + "@aws-sdk/client-dynamodb": { + "optional": true + }, + "@aws-sdk/lib-dynamodb": { + "optional": true + }, + "@duckdb/node-api": { + "optional": true + }, + "@google-cloud/cloud-sql-connector": { + "optional": true + }, + "@google-cloud/firestore": { + "optional": true + }, + "@libsql/client": { + "optional": true + }, + "mongodb": { + "optional": true + }, + "mssql": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "pg": { + "optional": true + }, + "redis": { + "optional": true + }, + "surrealdb": { + "optional": true + } } }, "node_modules/@babel/helper-string-parser": { diff --git a/package.json b/package.json index 8178ec0..bcf174a 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,9 @@ "server:prod": "node dist/server.js", "mcp:server": "tsx src/mcp/index.ts", "test": "vitest run", - "test:coverage": "vitest run --coverage" + "test:coverage": "vitest run --coverage", + "test:backends": "vitest run tests/store/backends.test.ts", + "db": "tsx src/index.ts db" }, "keywords": [ "chatgpt", @@ -26,7 +28,13 @@ "converter", "ai", "mcp", - "model-context-protocol" + "model-context-protocol", + "database", + "byodb", + "postgres", + "sqlite", + "mongodb", + "duckdb" ], "author": "", "license": "MIT", @@ -54,5 +62,57 @@ "tsx": "^4.23.12", "typescript": "^5.9.3", "vitest": "^4.1.8" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.0.0", + "@aws-sdk/lib-dynamodb": "^3.0.0", + "@duckdb/node-api": "^1.0.0", + "@google-cloud/cloud-sql-connector": "^1.0.0", + "@google-cloud/firestore": "^7.0.0", + "@libsql/client": "^0.15.0", + "mongodb": "^6.0.0", + "mssql": "^11.0.0", + "mysql2": "^3.0.0", + "pg": "^8.0.0", + "redis": "^5.0.0", + "surrealdb": "^2.0.0" + }, + "peerDependenciesMeta": { + "@aws-sdk/client-dynamodb": { + "optional": true + }, + "@aws-sdk/lib-dynamodb": { + "optional": true + }, + "@duckdb/node-api": { + "optional": true + }, + "@google-cloud/cloud-sql-connector": { + "optional": true + }, + "@google-cloud/firestore": { + "optional": true + }, + "@libsql/client": { + "optional": true + }, + "mongodb": { + "optional": true + }, + "mssql": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "pg": { + "optional": true + }, + "redis": { + "optional": true + }, + "surrealdb": { + "optional": true + } } } diff --git a/src/index.ts b/src/index.ts index 1a3ff52..f40b433 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,10 @@ import { ConversationNormalizer } from './parsers/normalizer.js'; import { MarkdownFormatter } from './formatters/markdown.js'; import { OllamaPreferenceAnalyzer } from './analyzers/ollama-preferences.js'; import { ensureDir, writeFile, copyImages } from './utils/file.js'; +import { createStore, ADAPTERS, isDriverInstalled } from './store/index.js'; +import { migrateStore } from './store/migrate.js'; +import { resolveDatabase, writeDatabaseUrl, clearDatabaseUrl } from './store/config.js'; +import { redactDsn } from './store/dsn.js'; import type { NormalizedConversation } from './parsers/types.js'; const program = new Command(); @@ -231,4 +235,117 @@ function generateUserProfile(userJson: any): string { } // Parse CLI arguments + +// --------------------------------------------------------------------------- +// db — inspect and switch the backing store (BYODB) +// --------------------------------------------------------------------------- + +const db = program.command('db').description('Manage the database backing the context store'); + +db.command('status') + .description('Show the current database and how it was configured') + .action(async () => { + const resolution = resolveDatabase(); + console.log(chalk.blue('\nopencontext database\n')); + console.log(` ${chalk.gray('Connection')} ${resolution.redacted}`); + console.log(` ${chalk.gray('Source')} ${resolution.source}`); + try { + const store = await createStore(resolution.url); + const [contexts, bubbles] = await Promise.all([ + store.listContexts(), + store.listBubbles(), + ]); + console.log(` ${chalk.gray('Adapter')} ${store.info.label}`); + console.log(` ${chalk.gray('Status')} ${chalk.green('connected')}`); + console.log(` ${chalk.gray('Contents')} ${contexts.length} contexts, ${bubbles.length} bubbles\n`); + await store.close(); + } catch (error) { + console.log(` ${chalk.gray('Status')} ${chalk.red('not connected')}`); + console.error(`\n${chalk.red(error instanceof Error ? error.message : String(error))}\n`); + process.exitCode = 1; + } + }); + +db.command('adapters') + .description('List every supported database and whether its driver is installed') + .action(async () => { + console.log(chalk.blue('\nSupported databases\n')); + for (const adapter of ADAPTERS) { + const installed = await isDriverInstalled(adapter.scheme); + const mark = installed ? chalk.green('✓') : chalk.gray('·'); + const need = adapter.packageName && !installed + ? chalk.gray(` npm install ${adapter.packageName}`) + : ''; + console.log(` ${mark} ${adapter.label.padEnd(24)} ${chalk.gray(adapter.example)}${need}`); + } + console.log(`\n ${chalk.green('✓')} ready ${chalk.gray('·')} driver not installed\n`); + }); + +db.command('test ') + .description('Try connecting to a database without saving it') + .action(async (url: string) => { + try { + const store = await createStore(url); + await store.ping(); + console.log(chalk.green(`\n✓ Connected to ${store.info.label} at ${store.info.target}\n`)); + await store.close(); + } catch (error) { + console.error(chalk.red(`\n✗ ${error instanceof Error ? error.message : String(error)}\n`)); + process.exit(1); + } + }); + +db.command('use ') + .description('Save a database connection as the default store') + .action(async (url: string) => { + try { + // Prove it works before persisting it, so a typo cannot leave the CLI and + // the MCP server pointed at something unusable. + const store = await createStore(url); + await store.ping(); + await store.close(); + writeDatabaseUrl(url); + console.log(chalk.green(`\n✓ Now using ${redactDsn(url)}\n`)); + } catch (error) { + console.error(chalk.red(`\n✗ ${error instanceof Error ? error.message : String(error)}\n`)); + process.exit(1); + } + }); + +db.command('reset') + .description('Forget the saved connection and go back to the default JSON store') + .action(() => { + clearDatabaseUrl(); + console.log(chalk.green(`\n✓ Reset to ${resolveDatabase().redacted}\n`)); + }); + +db.command('migrate') + .description('Copy all contexts and bubbles into another database') + .requiredOption('--to ', 'Target connection string') + .option('--from ', 'Source connection string (defaults to the current store)') + .option('--replace', 'Empty the target before copying', false) + .action(async (options: { to: string; from?: string; replace: boolean }) => { + const sourceUrl = options.from ?? resolveDatabase().url; + let source; + let target; + try { + source = await createStore(sourceUrl); + target = await createStore(options.to); + console.log(chalk.blue(`\nMigrating ${source.info.label} → ${target.info.label}\n`)); + + const result = await migrateStore(source, target, { + mode: options.replace ? 'replace' : 'copy', + }); + console.log(chalk.green(`✓ Copied ${result.contexts} contexts and ${result.bubbles} bubbles`)); + console.log(chalk.gray(` The source store was not modified.\n`)); + } catch (error) { + console.error(chalk.red(`\n✗ ${error instanceof Error ? error.message : String(error)}\n`)); + process.exit(1); + } finally { + await source?.close().catch(() => undefined); + await target?.close().catch(() => undefined); + } + }); + + program.parse(); diff --git a/src/mcp/index.ts b/src/mcp/index.ts index 97cc68a..84cd2ca 100644 --- a/src/mcp/index.ts +++ b/src/mcp/index.ts @@ -3,9 +3,9 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; import { createMcpServer } from './server.js'; -const storePath = process.env.OPENCONTEXT_STORE_PATH || undefined; - -const server = createMcpServer(storePath); +// The store is resolved from the environment and saved config inside the server, +// so nothing needs to be passed in here. +const server = createMcpServer(); const transport = new StdioServerTransport(); server.connect(transport).catch((error) => { diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 31beac6..0f0a720 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -1,9 +1,19 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; -import { createStore } from './store.js'; +import { createStoreManager } from '../store/manager.js'; -export function createMcpServer(storePath?: string) { - const store = createStore(storePath); +/** + * @param databaseUrl Optional connection string. When omitted the store is + * resolved from OPENCONTEXT_DB_URL, then the saved config, then the legacy + * OPENCONTEXT_STORE_PATH, then the default JSON file. + */ +export function createMcpServer(databaseUrl?: string) { + const manager = createStoreManager(); + // The backend connects on first tool call rather than at construction, so an + // unreachable database surfaces as a tool error instead of preventing the MCP + // server from starting at all. + const store = () => + databaseUrl ? manager.reconnect(databaseUrl).then(() => manager.get()) : manager.get(); const server = new McpServer({ name: 'opencontext', @@ -33,7 +43,7 @@ export function createMcpServer(storePath?: string) { .describe('ID of the bubble (project) to associate this context with'), }, async (args) => { - const entry = store.saveContext( + const entry = await (await store()).saveContext( args.content, args.tags || [], args.source || 'chat', @@ -57,7 +67,7 @@ export function createMcpServer(storePath?: string) { query: z.string().describe('Search query to find matching contexts'), }, async (args) => { - const results = store.recallContext(args.query); + const results = await (await store()).recallContext(args.query); if (results.length === 0) { return { content: [ @@ -95,7 +105,7 @@ export function createMcpServer(storePath?: string) { .describe('Filter by tag (e.g. "preference", "code")'), }, async (args) => { - const results = store.listContexts(args.tag); + const results = await (await store()).listContexts(args.tag); if (results.length === 0) { return { content: [ @@ -132,7 +142,7 @@ export function createMcpServer(storePath?: string) { id: z.string().describe('The ID of the context to delete'), }, async (args) => { - const deleted = store.deleteContext(args.id); + const deleted = await (await store()).deleteContext(args.id); return { content: [ { @@ -155,7 +165,7 @@ export function createMcpServer(storePath?: string) { .describe('Space-separated search terms (all must match)'), }, async (args) => { - const results = store.searchContexts(args.query); + const results = await (await store()).searchContexts(args.query); if (results.length === 0) { return { content: [ @@ -200,7 +210,7 @@ export function createMcpServer(storePath?: string) { .describe('Bubble ID to assign (null to unassign from bubble)'), }, async (args) => { - const updated = store.updateContext(args.id, args.content, args.tags, args.bubbleId); + const updated = await (await store()).updateContext(args.id, args.content, args.tags, args.bubbleId); if (!updated) { return { content: [ @@ -237,7 +247,7 @@ export function createMcpServer(storePath?: string) { .describe('Optional description of what this bubble is for'), }, async (args) => { - const bubble = store.createBubble(args.name, args.description); + const bubble = await (await store()).createBubble(args.name, args.description); return { content: [ { @@ -254,18 +264,21 @@ export function createMcpServer(storePath?: string) { 'List all bubbles (project workspaces).', {}, async () => { - const bubbles = store.listBubbles(); + const bubbles = await (await store()).listBubbles(); if (bubbles.length === 0) { return { content: [{ type: 'text' as const, text: 'No bubbles created yet.' }], }; } - const formatted = bubbles - .map((b) => { - const contexts = store.listContextsByBubble(b.id); - return `[${b.id}] ${b.name}${b.description ? ` — ${b.description}` : ''} (${contexts.length} context${contexts.length === 1 ? '' : 's'})`; - }) - .join('\n'); + const db = await store(); + const formatted = ( + await Promise.all( + bubbles.map(async (b) => { + const contexts = await db.listContextsByBubble(b.id); + return `[${b.id}] ${b.name}${b.description ? ` — ${b.description}` : ''} (${contexts.length} context${contexts.length === 1 ? '' : 's'})`; + }), + ) + ).join('\n'); return { content: [{ type: 'text' as const, text: `${bubbles.length} bubble(s):\n\n${formatted}` }], }; @@ -279,13 +292,13 @@ export function createMcpServer(storePath?: string) { id: z.string().describe('The ID of the bubble'), }, async (args) => { - const bubble = store.getBubble(args.id); + const bubble = await (await store()).getBubble(args.id); if (!bubble) { return { content: [{ type: 'text' as const, text: `No bubble found with ID "${args.id}".` }], }; } - const contexts = store.listContextsByBubble(args.id); + const contexts = await (await store()).listContextsByBubble(args.id); const ctxText = contexts.length === 0 ? 'No contexts in this bubble.' @@ -318,7 +331,7 @@ export function createMcpServer(storePath?: string) { .describe('New description (omit to leave unchanged)'), }, async (args) => { - const updated = store.updateBubble(args.id, args.name, args.description); + const updated = await (await store()).updateBubble(args.id, args.name, args.description); if (!updated) { return { content: [{ type: 'text' as const, text: `No bubble found with ID "${args.id}".` }], @@ -346,7 +359,7 @@ export function createMcpServer(storePath?: string) { .describe('If true, also delete all contexts inside the bubble (default: false)'), }, async (args) => { - const deleted = store.deleteBubble(args.id, args.deleteContexts ?? false); + const deleted = await (await store()).deleteBubble(args.id, args.deleteContexts ?? false); return { content: [ { diff --git a/src/mcp/store.ts b/src/mcp/store.ts index 56b58a3..8617050 100644 --- a/src/mcp/store.ts +++ b/src/mcp/store.ts @@ -1,229 +1,10 @@ -import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs'; -import { join, dirname } from 'path'; -import { randomUUID } from 'crypto'; -import { ContextEntry, ContextStore, Bubble } from './types.js'; - -const STORE_VERSION = 1; - -function getDefaultStorePath(): string { - const home = process.env.HOME || process.env.USERPROFILE || '.'; - return join(home, '.opencontext', 'contexts.json'); -} - -export function createStore(storePath?: string) { - const filePath = storePath || getDefaultStorePath(); - - function load(): ContextStore { - if (!existsSync(filePath)) { - return { version: STORE_VERSION, entries: [], bubbles: [] }; - } - const raw = readFileSync(filePath, 'utf-8'); - const parsed = JSON.parse(raw) as ContextStore; - // Migrate stores that predate the bubbles field - if (!parsed.bubbles) { - parsed.bubbles = []; - } - return parsed; - } - - function save(store: ContextStore): void { - const directory = dirname(filePath); - if (!existsSync(directory)) { - mkdirSync(directory, { recursive: true }); - } - writeFileSync(filePath, JSON.stringify(store, null, 2), 'utf-8'); - } - - // --------------------------------------------------------------------------- - // Context CRUD - // --------------------------------------------------------------------------- - - function saveContext( - content: string, - tags: string[] = [], - source: string = 'chat', - bubbleId?: string, - ): ContextEntry { - const store = load(); - const now = new Date().toISOString(); - const entry: ContextEntry = { - id: randomUUID(), - content, - tags, - source, - createdAt: now, - updatedAt: now, - }; - if (bubbleId !== undefined) { - entry.bubbleId = bubbleId; - } - store.entries.push(entry); - save(store); - return entry; - } - - function recallContext(query: string): ContextEntry[] { - const store = load(); - const lowerQuery = query.toLowerCase(); - return store.entries.filter( - (entry) => - entry.content.toLowerCase().includes(lowerQuery) || - entry.tags.some((tag) => tag.toLowerCase().includes(lowerQuery)), - ); - } - - function listContexts(tag?: string): ContextEntry[] { - const store = load(); - if (!tag) { - return store.entries; - } - const lowerTag = tag.toLowerCase(); - return store.entries.filter((entry) => - entry.tags.some((t) => t.toLowerCase() === lowerTag), - ); - } - - function listContextsByBubble(bubbleId: string): ContextEntry[] { - const store = load(); - return store.entries.filter((entry) => entry.bubbleId === bubbleId); - } - - function deleteContext(id: string): boolean { - const store = load(); - const initialLength = store.entries.length; - store.entries = store.entries.filter((entry) => entry.id !== id); - if (store.entries.length < initialLength) { - save(store); - return true; - } - return false; - } - - function searchContexts(query: string): ContextEntry[] { - const store = load(); - const lowerQuery = query.toLowerCase(); - const terms = lowerQuery.split(/\s+/).filter(Boolean); - return store.entries.filter((entry) => { - const text = `${entry.content} ${entry.tags.join(' ')} ${entry.source}`.toLowerCase(); - return terms.every((term) => text.includes(term)); - }); - } - - function getContext(id: string): ContextEntry | undefined { - const store = load(); - return store.entries.find((entry) => entry.id === id); - } - - function updateContext( - id: string, - content: string, - tags?: string[], - bubbleId?: string | null, - ): ContextEntry | undefined { - const store = load(); - const entry = store.entries.find((e) => e.id === id); - if (!entry) { - return undefined; - } - entry.content = content; - if (tags !== undefined) { - entry.tags = tags; - } - if (bubbleId !== undefined) { - if (bubbleId === null) { - delete entry.bubbleId; - } else { - entry.bubbleId = bubbleId; - } - } - entry.updatedAt = new Date().toISOString(); - save(store); - return entry; - } - - // --------------------------------------------------------------------------- - // Bubble CRUD - // --------------------------------------------------------------------------- - - function createBubble(name: string, description?: string): Bubble { - const store = load(); - const now = new Date().toISOString(); - const bubble: Bubble = { - id: randomUUID(), - name, - createdAt: now, - updatedAt: now, - }; - if (description !== undefined) { - bubble.description = description; - } - store.bubbles.push(bubble); - save(store); - return bubble; - } - - function listBubbles(): Bubble[] { - return load().bubbles; - } - - function getBubble(id: string): Bubble | undefined { - return load().bubbles.find((b) => b.id === id); - } - - function updateBubble(id: string, name: string, description?: string): Bubble | undefined { - const store = load(); - const bubble = store.bubbles.find((b) => b.id === id); - if (!bubble) { - return undefined; - } - bubble.name = name; - if (description !== undefined) { - bubble.description = description; - } - bubble.updatedAt = new Date().toISOString(); - save(store); - return bubble; - } - - function deleteBubble(id: string, deleteContexts = false): boolean { - const store = load(); - const initialLength = store.bubbles.length; - store.bubbles = store.bubbles.filter((b) => b.id !== id); - if (store.bubbles.length === initialLength) { - return false; - } - if (deleteContexts) { - store.entries = store.entries.filter((e) => e.bubbleId !== id); - } else { - // Unassign contexts from the deleted bubble - store.entries.forEach((e) => { - if (e.bubbleId === id) { - delete e.bubbleId; - } - }); - } - save(store); - return true; - } - - return { - // contexts - saveContext, - recallContext, - listContexts, - listContextsByBubble, - deleteContext, - searchContexts, - getContext, - updateContext, - // bubbles - createBubble, - listBubbles, - getBubble, - updateBubble, - deleteBubble, - // internals - load, - filePath, - }; -} +/** + * @deprecated The context store now lives in `src/store/`, where it is one of + * several interchangeable backends rather than a hard-coded JSON file. + * + * This module re-exports the new entry points so that anything importing the old + * path keeps resolving. New code should import from `../store/index.js`. + */ +export { createStore } from '../store/index.js'; +export { createStoreManager } from '../store/manager.js'; +export type { ContextStoreAdapter } from '../store/types.js'; diff --git a/src/server.ts b/src/server.ts index d671ca3..e7e8983 100644 --- a/src/server.ts +++ b/src/server.ts @@ -9,7 +9,12 @@ import { ZipExtractor } from './extractor.js'; import { ChatGPTParser } from './parsers/chatgpt.js'; import { ConversationNormalizer } from './parsers/normalizer.js'; import { OllamaPreferenceAnalyzer } from './analyzers/ollama-preferences.js'; -import { createStore } from './mcp/store.js'; +import { createStoreManager } from './store/manager.js'; +import { migrateStore } from './store/migrate.js'; +import { createStore, ADAPTERS, isDriverInstalled } from './store/index.js'; +import { parseDsn, redactDsn } from './store/dsn.js'; +import { resolveDatabase, clearDatabaseUrl, getDefaultJsonPath } from './store/config.js'; +import { InvalidDsnError, DriverNotInstalledError } from './store/types.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -25,14 +30,16 @@ const upload = multer({ dest: uploadDir }); // Ollama host — defaults to host.docker.internal so containers reach the host machine const OLLAMA_HOST = process.env.OLLAMA_HOST ?? 'http://host.docker.internal:11434'; -// Context store -const storePath = - process.env.OPENCONTEXT_STORE_PATH ?? - path.join(os.homedir(), '.opencontext', 'contexts.json'); -const store = createStore(storePath); +// Context store — a pluggable backend resolved from env, saved config, or the +// legacy store path. Connects lazily so importing this module stays synchronous. +const storeManager = createStoreManager(); +const store = () => storeManager.get(); -// Preferences files live alongside the context store -const prefsDir = path.dirname(storePath); +// Preferences remain files on disk regardless of which database backs the +// contexts, because Claude reads them straight from the filesystem. +const prefsDir = path.dirname( + process.env.OPENCONTEXT_STORE_PATH ?? getDefaultJsonPath(), +); const prefsJsonPath = path.join(prefsDir, 'preferences.json'); const prefsMdPath = path.join(prefsDir, 'preferences.md'); const memoryMdPath = path.join(prefsDir, 'memory.md'); @@ -120,7 +127,7 @@ if (fs.existsSync(publicDir)) { // --------------------------------------------------------------------------- app.get('/api/health', (_req: Request, res: Response) => { - res.json({ status: 'ok', ollamaHost: OLLAMA_HOST, store: storePath }); + res.json({ status: 'ok', ollamaHost: OLLAMA_HOST, store: resolveDatabase().redacted }); }); // --------------------------------------------------------------------------- @@ -245,12 +252,12 @@ app.put('/api/preferences', (req: Request, res: Response) => { // Contexts — CRUD for the MCP context store // --------------------------------------------------------------------------- -app.get('/api/contexts', (req: Request, res: Response) => { +app.get('/api/contexts', async (req: Request, res: Response) => { const tag = req.query.tag as string | undefined; - res.json(store.listContexts(tag)); + res.json(await (await store()).listContexts(tag)); }); -app.post('/api/contexts', (req: Request, res: Response) => { +app.post('/api/contexts', async (req: Request, res: Response) => { const { content, tags, source, bubbleId } = req.body as { content: string; tags?: string[]; @@ -261,20 +268,20 @@ app.post('/api/contexts', (req: Request, res: Response) => { res.status(400).json({ error: 'content is required' }); return; } - res.status(201).json(store.saveContext(content, tags, source, bubbleId)); + res.status(201).json(await (await store()).saveContext(content, tags, source, bubbleId)); }); -app.get('/api/contexts/search', (req: Request, res: Response) => { +app.get('/api/contexts/search', async (req: Request, res: Response) => { const q = req.query.q as string; if (!q) { res.status(400).json({ error: 'q query param required' }); return; } - res.json(store.searchContexts(q)); + res.json(await (await store()).searchContexts(q)); }); -app.get('/api/contexts/:id', (req: Request, res: Response) => { - const entry = store.getContext(req.params['id'] as string); +app.get('/api/contexts/:id', async (req: Request, res: Response) => { + const entry = await (await store()).getContext(req.params['id'] as string); if (!entry) { res.status(404).json({ error: 'Not found' }); return; @@ -282,7 +289,7 @@ app.get('/api/contexts/:id', (req: Request, res: Response) => { res.json(entry); }); -app.put('/api/contexts/:id', (req: Request, res: Response) => { +app.put('/api/contexts/:id', async (req: Request, res: Response) => { const { content, tags, bubbleId } = req.body as { content: string; tags?: string[]; @@ -292,7 +299,7 @@ app.put('/api/contexts/:id', (req: Request, res: Response) => { res.status(400).json({ error: 'content is required' }); return; } - const updated = store.updateContext(req.params['id'] as string, content, tags, bubbleId); + const updated = await (await store()).updateContext(req.params['id'] as string, content, tags, bubbleId); if (!updated) { res.status(404).json({ error: 'Not found' }); return; @@ -300,8 +307,8 @@ app.put('/api/contexts/:id', (req: Request, res: Response) => { res.json(updated); }); -app.delete('/api/contexts/:id', (req: Request, res: Response) => { - const deleted = store.deleteContext(req.params['id'] as string); +app.delete('/api/contexts/:id', async (req: Request, res: Response) => { + const deleted = await (await store()).deleteContext(req.params['id'] as string); if (!deleted) { res.status(404).json({ error: 'Not found' }); return; @@ -313,52 +320,55 @@ app.delete('/api/contexts/:id', (req: Request, res: Response) => { // Bubbles — CRUD for project workspaces // --------------------------------------------------------------------------- -app.get('/api/bubbles', (_req: Request, res: Response) => { - const bubbles = store.listBubbles(); - const withCounts = bubbles.map((b) => ({ - ...b, - contextCount: store.listContextsByBubble(b.id).length, - })); +app.get('/api/bubbles', async (_req: Request, res: Response) => { + const db = await store(); + const bubbles = await db.listBubbles(); + const withCounts = await Promise.all( + bubbles.map(async (b) => ({ + ...b, + contextCount: (await db.listContextsByBubble(b.id)).length, + })), + ); res.json(withCounts); }); -app.post('/api/bubbles', (req: Request, res: Response) => { +app.post('/api/bubbles', async (req: Request, res: Response) => { const { name, description } = req.body as { name: string; description?: string }; if (!name) { res.status(400).json({ error: 'name is required' }); return; } - res.status(201).json(store.createBubble(name, description)); + res.status(201).json(await (await store()).createBubble(name, description)); }); -app.get('/api/bubbles/:id', (req: Request, res: Response) => { - const bubble = store.getBubble(req.params['id'] as string); +app.get('/api/bubbles/:id', async (req: Request, res: Response) => { + const bubble = await (await store()).getBubble(req.params['id'] as string); if (!bubble) { res.status(404).json({ error: 'Not found' }); return; } res.json({ ...bubble, - contextCount: store.listContextsByBubble(bubble.id).length, + contextCount: (await (await store()).listContextsByBubble(bubble.id)).length, }); }); -app.get('/api/bubbles/:id/contexts', (req: Request, res: Response) => { - const bubble = store.getBubble(req.params['id'] as string); +app.get('/api/bubbles/:id/contexts', async (req: Request, res: Response) => { + const bubble = await (await store()).getBubble(req.params['id'] as string); if (!bubble) { res.status(404).json({ error: 'Not found' }); return; } - res.json(store.listContextsByBubble(req.params['id'] as string)); + res.json(await (await store()).listContextsByBubble(req.params['id'] as string)); }); -app.put('/api/bubbles/:id', (req: Request, res: Response) => { +app.put('/api/bubbles/:id', async (req: Request, res: Response) => { const { name, description } = req.body as { name: string; description?: string }; if (!name) { res.status(400).json({ error: 'name is required' }); return; } - const updated = store.updateBubble(req.params['id'] as string, name, description); + const updated = await (await store()).updateBubble(req.params['id'] as string, name, description); if (!updated) { res.status(404).json({ error: 'Not found' }); return; @@ -366,9 +376,9 @@ app.put('/api/bubbles/:id', (req: Request, res: Response) => { res.json(updated); }); -app.delete('/api/bubbles/:id', (req: Request, res: Response) => { +app.delete('/api/bubbles/:id', async (req: Request, res: Response) => { const deleteContexts = req.query['deleteContexts'] === 'true'; - const deleted = store.deleteBubble(req.params['id'] as string, deleteContexts); + const deleted = await (await store()).deleteBubble(req.params['id'] as string, deleteContexts); if (!deleted) { res.status(404).json({ error: 'Not found' }); return; @@ -376,6 +386,144 @@ app.delete('/api/bubbles/:id', (req: Request, res: Response) => { res.status(204).send(); }); + +// --------------------------------------------------------------------------- +// Database — inspect, test, switch and migrate the backing store (BYODB) +// --------------------------------------------------------------------------- + +/** Turn a store failure into a useful message without leaking credentials. */ +function describeStoreError(error: unknown): { status: number; message: string } { + if (error instanceof InvalidDsnError) { + return { status: 400, message: error.message }; + } + if (error instanceof DriverNotInstalledError) { + return { status: 400, message: error.message }; + } + const raw = error instanceof Error ? error.message : String(error); + // The driver's own message is the only useful diagnostic for a refused + // connection, but it can echo the connection string back — so redact it. + return { status: 502, message: redactDsn(raw) }; +} + +app.get('/api/db/status', async (_req: Request, res: Response) => { + const resolution = resolveDatabase(); + try { + const db = await store(); + const [contexts, bubbles] = await Promise.all([db.listContexts(), db.listBubbles()]); + res.json({ + connected: true, + adapter: db.info, + source: resolution.source, + locked: resolution.locked, + url: resolution.redacted, + counts: { contexts: contexts.length, bubbles: bubbles.length }, + }); + } catch (error) { + const { message } = describeStoreError(error); + res.json({ + connected: false, + adapter: null, + source: resolution.source, + locked: resolution.locked, + url: resolution.redacted, + counts: null, + error: message, + }); + } +}); + +app.get('/api/db/adapters', async (_req: Request, res: Response) => { + const adapters = await Promise.all( + ADAPTERS.map(async (adapter) => ({ + ...adapter, + installed: await isDriverInstalled(adapter.scheme), + })), + ); + res.json(adapters); +}); + +app.post('/api/db/test', async (req: Request, res: Response) => { + const { url } = req.body as { url?: string }; + if (!url) { + res.status(400).json({ ok: false, error: 'url is required' }); + return; + } + let candidate; + try { + // Open, ping, and close again — a test must never leave a connection behind + // or disturb the store currently in use. + candidate = await createStore(url); + await candidate.ping(); + res.json({ ok: true, adapter: candidate.info }); + } catch (error) { + const { status, message } = describeStoreError(error); + res.status(status).json({ ok: false, error: message }); + } finally { + await candidate?.close().catch(() => undefined); + } +}); + +app.put('/api/db/config', async (req: Request, res: Response) => { + const { url } = req.body as { url?: string }; + if (!url) { + res.status(400).json({ error: 'url is required' }); + return; + } + if (resolveDatabase().locked) { + res.status(409).json({ + error: + 'The database is set by the OPENCONTEXT_DB_URL environment variable, ' + + 'which takes precedence over saved settings. Unset it to change the store here.', + }); + return; + } + try { + parseDsn(url); + // reconnect keeps the previous store if the new one fails to open, so a bad + // connection string cannot take the running server down. + const info = await storeManager.reconnect(url, { persist: true }); + res.json({ ok: true, adapter: info }); + } catch (error) { + const { status, message } = describeStoreError(error); + res.status(status).json({ ok: false, error: message }); + } +}); + +app.delete('/api/db/config', async (_req: Request, res: Response) => { + if (resolveDatabase().locked) { + res.status(409).json({ error: 'The database is pinned by an environment variable.' }); + return; + } + clearDatabaseUrl(); + try { + const info = await storeManager.reconnect(resolveDatabase().url); + res.json({ ok: true, adapter: info }); + } catch (error) { + const { status, message } = describeStoreError(error); + res.status(status).json({ ok: false, error: message }); + } +}); + +app.post('/api/db/migrate', async (req: Request, res: Response) => { + const { url, mode } = req.body as { url?: string; mode?: 'copy' | 'replace' }; + if (!url) { + res.status(400).json({ error: 'url is required' }); + return; + } + let target; + try { + target = await createStore(url); + // The source is only read, so a failure here cannot damage existing data. + const result = await migrateStore(await store(), target, { mode: mode ?? 'copy' }); + res.json({ ok: true, ...result, target: target.info }); + } catch (error) { + const { status, message } = describeStoreError(error); + res.status(status).json({ ok: false, error: message }); + } finally { + await target?.close().catch(() => undefined); + } +}); + // --------------------------------------------------------------------------- // SPA fallback — all non-API routes serve the React app // --------------------------------------------------------------------------- @@ -401,7 +549,7 @@ if (process.env.NODE_ENV !== 'test') { app.listen(PORT, '0.0.0.0', () => { console.log(`opencontext server → http://0.0.0.0:${PORT}`); console.log(`Ollama host → ${OLLAMA_HOST}`); - console.log(`Context store → ${storePath}`); + console.log(`Context store → ${resolveDatabase().redacted}`); console.log(`UI → ${fs.existsSync(publicDir) ? 'served from /public' : 'not built'}`); }); } diff --git a/src/store/adapters/document.ts b/src/store/adapters/document.ts new file mode 100644 index 0000000..11cfdd2 --- /dev/null +++ b/src/store/adapters/document.ts @@ -0,0 +1,238 @@ +import { randomUUID } from 'crypto'; +import type { + ContextStoreAdapter, + AdapterInfo, + ContextEntry, + Bubble, +} from '../types.js'; + +export type Collection = 'contexts' | 'bubbles'; + +export type Document = Record; + +/** + * The minimum a document or key-value store must provide. + * + * Everything else — search, tag filtering, ordering, bubble cascade — is + * implemented once in `createDocumentAdapter`, so adding a new NoSQL backend + * means writing six small methods rather than the whole storage contract. + */ +export interface DocumentDriver { + connect(): Promise; + close(): Promise; + ping(): Promise; + get(collection: Collection, id: string): Promise; + put(collection: Collection, id: string, document: Document): Promise; + remove(collection: Collection, id: string): Promise; + list(collection: Collection): Promise; +} + +function byCreatedThenId(a: T, b: T): number { + return a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id); +} + +function toEntry(document: Document): ContextEntry { + const entry: ContextEntry = { + id: document.id as string, + content: document.content as string, + tags: (document.tags as string[]) ?? [], + source: document.source as string, + createdAt: document.createdAt as string, + updatedAt: document.updatedAt as string, + }; + if (document.bubbleId !== undefined && document.bubbleId !== null) { + entry.bubbleId = document.bubbleId as string; + } + return entry; +} + +function toBubble(document: Document): Bubble { + const bubble: Bubble = { + id: document.id as string, + name: document.name as string, + createdAt: document.createdAt as string, + updatedAt: document.updatedAt as string, + }; + if (document.description !== undefined && document.description !== null) { + bubble.description = document.description as string; + } + return bubble; +} + +/** + * The storage contract over any document or key-value store. + * + * Predicates are evaluated in memory rather than pushed down, because the + * backends behind this interface either cannot express case-insensitive + * substring search at all (DynamoDB, Redis) or express it in mutually + * incompatible ways (Mongo regex, Firestore's lack of one). Evaluating in one + * place keeps search semantics byte-identical to the SQL and JSON adapters, + * which is what makes the shared conformance suite meaningful. + * + * The cost is real: search reads the whole context collection. For a store large + * enough for that to hurt, a SQL backend is the better choice, and the README + * says so. + */ +export function createDocumentAdapter( + driver: DocumentDriver, + info: AdapterInfo, +): ContextStoreAdapter { + async function allEntries(): Promise { + const documents = await driver.list('contexts'); + return documents.map(toEntry).sort(byCreatedThenId); + } + + return { + info, + + connect: () => driver.connect(), + close: () => driver.close(), + ping: () => driver.ping(), + + // ----------------------------------------------------------------------- + // Contexts + // ----------------------------------------------------------------------- + + async saveContext(content, tags = [], source = 'chat', bubbleId) { + const now = new Date().toISOString(); + const entry: ContextEntry = { + id: randomUUID(), + content, + tags, + source, + createdAt: now, + updatedAt: now, + }; + if (bubbleId !== undefined) { + entry.bubbleId = bubbleId; + } + await driver.put('contexts', entry.id, { ...entry }); + return entry; + }, + + async recallContext(query) { + const needle = query.toLowerCase(); + return (await allEntries()).filter( + (entry) => + entry.content.toLowerCase().includes(needle) || + entry.tags.some((tag) => tag.toLowerCase().includes(needle)), + ); + }, + + async listContexts(tag) { + const entries = await allEntries(); + if (!tag) { + return entries; + } + const lower = tag.toLowerCase(); + return entries.filter((entry) => entry.tags.some((t) => t.toLowerCase() === lower)); + }, + + async listContextsByBubble(bubbleId) { + return (await allEntries()).filter((entry) => entry.bubbleId === bubbleId); + }, + + async getContext(id) { + const document = await driver.get('contexts', id); + return document ? toEntry(document) : undefined; + }, + + async updateContext(id, content, tags, bubbleId) { + const existing = await driver.get('contexts', id); + if (!existing) { + return undefined; + } + const entry = toEntry(existing); + entry.content = content; + if (tags !== undefined) { + entry.tags = tags; + } + if (bubbleId === null) { + delete entry.bubbleId; + } else if (bubbleId !== undefined) { + entry.bubbleId = bubbleId; + } + entry.updatedAt = new Date().toISOString(); + await driver.put('contexts', id, { ...entry }); + return entry; + }, + + async deleteContext(id) { + if (!(await driver.get('contexts', id))) { + return false; + } + await driver.remove('contexts', id); + return true; + }, + + async searchContexts(query) { + const terms = query.toLowerCase().split(/\s+/).filter(Boolean); + const entries = await allEntries(); + if (terms.length === 0) { + return entries; + } + return entries.filter((entry) => { + const haystack = + `${entry.content} ${entry.tags.join(' ')} ${entry.source}`.toLowerCase(); + return terms.every((term) => haystack.includes(term)); + }); + }, + + // ----------------------------------------------------------------------- + // Bubbles + // ----------------------------------------------------------------------- + + async createBubble(name, description) { + const now = new Date().toISOString(); + const bubble: Bubble = { id: randomUUID(), name, createdAt: now, updatedAt: now }; + if (description !== undefined) { + bubble.description = description; + } + await driver.put('bubbles', bubble.id, { ...bubble }); + return bubble; + }, + + async listBubbles() { + const documents = await driver.list('bubbles'); + return documents.map(toBubble).sort(byCreatedThenId); + }, + + async getBubble(id) { + const document = await driver.get('bubbles', id); + return document ? toBubble(document) : undefined; + }, + + async updateBubble(id, name, description) { + const existing = await driver.get('bubbles', id); + if (!existing) { + return undefined; + } + const bubble = toBubble(existing); + bubble.name = name; + if (description !== undefined) { + bubble.description = description; + } + bubble.updatedAt = new Date().toISOString(); + await driver.put('bubbles', id, { ...bubble }); + return bubble; + }, + + async deleteBubble(id, deleteContexts = false) { + if (!(await driver.get('bubbles', id))) { + return false; + } + await driver.remove('bubbles', id); + + const affected = (await allEntries()).filter((entry) => entry.bubbleId === id); + for (const entry of affected) { + if (deleteContexts) { + await driver.remove('contexts', entry.id); + } else { + delete entry.bubbleId; + await driver.put('contexts', entry.id, { ...entry }); + } + } + return true; + }, + }; +} diff --git a/src/store/adapters/json.ts b/src/store/adapters/json.ts new file mode 100644 index 0000000..fd1a432 --- /dev/null +++ b/src/store/adapters/json.ts @@ -0,0 +1,237 @@ +import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs'; +import { dirname } from 'path'; +import { randomUUID } from 'crypto'; +import type { + ContextStoreAdapter, + AdapterInfo, + ContextEntry, + Bubble, +} from '../types.js'; +import type { ParsedDsn } from '../dsn.js'; + +const STORE_VERSION = 1; + +interface JsonStoreFile { + version: number; + entries: ContextEntry[]; + bubbles: Bubble[]; +} + +/** The documented ordering contract: createdAt ascending, then id ascending. */ +function byCreatedThenId(a: T, b: T): number { + return a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id); +} + +/** + * The original file-backed store, now behind the adapter interface. + * + * Reads and writes the whole document per operation, exactly as before. That is + * fine for the default single-user case and is precisely the limitation the SQL + * adapters exist to escape. + */ +export function createJsonAdapter(dsn: ParsedDsn): ContextStoreAdapter { + const filePath = dsn.path!; + + const info: AdapterInfo = { + scheme: 'json', + label: 'JSON file', + target: filePath, + remote: false, + }; + + function load(): JsonStoreFile { + if (!existsSync(filePath)) { + return { version: STORE_VERSION, entries: [], bubbles: [] }; + } + const parsed = JSON.parse(readFileSync(filePath, 'utf-8')) as JsonStoreFile; + // Migrate stores that predate the bubbles field + if (!parsed.bubbles) { + parsed.bubbles = []; + } + if (!parsed.entries) { + parsed.entries = []; + } + return parsed; + } + + function save(store: JsonStoreFile): void { + const directory = dirname(filePath); + if (directory && !existsSync(directory)) { + mkdirSync(directory, { recursive: true }); + } + writeFileSync(filePath, JSON.stringify(store, null, 2), 'utf-8'); + } + + function sortedEntries(store: JsonStoreFile): ContextEntry[] { + return [...store.entries].sort(byCreatedThenId); + } + + return { + info, + + async connect() { + const directory = dirname(filePath); + if (directory && !existsSync(directory)) { + mkdirSync(directory, { recursive: true }); + } + }, + + async close() { + // Nothing to release — every operation opens and closes the file itself. + }, + + async ping() { + load(); + }, + + // ----------------------------------------------------------------------- + // Contexts + // ----------------------------------------------------------------------- + + async saveContext(content, tags = [], source = 'chat', bubbleId) { + const store = load(); + const now = new Date().toISOString(); + const entry: ContextEntry = { + id: randomUUID(), + content, + tags, + source, + createdAt: now, + updatedAt: now, + }; + if (bubbleId !== undefined) { + entry.bubbleId = bubbleId; + } + store.entries.push(entry); + save(store); + return entry; + }, + + async recallContext(query) { + const lower = query.toLowerCase(); + return sortedEntries(load()).filter( + (entry) => + entry.content.toLowerCase().includes(lower) || + entry.tags.some((tag) => tag.toLowerCase().includes(lower)), + ); + }, + + async listContexts(tag) { + const entries = sortedEntries(load()); + if (!tag) { + return entries; + } + const lowerTag = tag.toLowerCase(); + return entries.filter((entry) => entry.tags.some((t) => t.toLowerCase() === lowerTag)); + }, + + async listContextsByBubble(bubbleId) { + return sortedEntries(load()).filter((entry) => entry.bubbleId === bubbleId); + }, + + async getContext(id) { + return load().entries.find((entry) => entry.id === id); + }, + + async updateContext(id, content, tags, bubbleId) { + const store = load(); + const entry = store.entries.find((e) => e.id === id); + if (!entry) { + return undefined; + } + entry.content = content; + if (tags !== undefined) { + entry.tags = tags; + } + if (bubbleId !== undefined) { + if (bubbleId === null) { + delete entry.bubbleId; + } else { + entry.bubbleId = bubbleId; + } + } + entry.updatedAt = new Date().toISOString(); + save(store); + return entry; + }, + + async deleteContext(id) { + const store = load(); + const before = store.entries.length; + store.entries = store.entries.filter((entry) => entry.id !== id); + if (store.entries.length === before) { + return false; + } + save(store); + return true; + }, + + async searchContexts(query) { + const terms = query.toLowerCase().split(/\s+/).filter(Boolean); + return sortedEntries(load()).filter((entry) => { + const haystack = + `${entry.content} ${entry.tags.join(' ')} ${entry.source}`.toLowerCase(); + return terms.every((term) => haystack.includes(term)); + }); + }, + + // ----------------------------------------------------------------------- + // Bubbles + // ----------------------------------------------------------------------- + + async createBubble(name, description) { + const store = load(); + const now = new Date().toISOString(); + const bubble: Bubble = { id: randomUUID(), name, createdAt: now, updatedAt: now }; + if (description !== undefined) { + bubble.description = description; + } + store.bubbles.push(bubble); + save(store); + return bubble; + }, + + async listBubbles() { + return [...load().bubbles].sort(byCreatedThenId); + }, + + async getBubble(id) { + return load().bubbles.find((bubble) => bubble.id === id); + }, + + async updateBubble(id, name, description) { + const store = load(); + const bubble = store.bubbles.find((b) => b.id === id); + if (!bubble) { + return undefined; + } + bubble.name = name; + if (description !== undefined) { + bubble.description = description; + } + bubble.updatedAt = new Date().toISOString(); + save(store); + return bubble; + }, + + async deleteBubble(id, deleteContexts = false) { + const store = load(); + const before = store.bubbles.length; + store.bubbles = store.bubbles.filter((bubble) => bubble.id !== id); + if (store.bubbles.length === before) { + return false; + } + if (deleteContexts) { + store.entries = store.entries.filter((entry) => entry.bubbleId !== id); + } else { + store.entries.forEach((entry) => { + if (entry.bubbleId === id) { + delete entry.bubbleId; + } + }); + } + save(store); + return true; + }, + }; +} diff --git a/src/store/adapters/sql.ts b/src/store/adapters/sql.ts new file mode 100644 index 0000000..1e58df7 --- /dev/null +++ b/src/store/adapters/sql.ts @@ -0,0 +1,530 @@ +import { randomUUID } from 'crypto'; +import type { + ContextStoreAdapter, + AdapterInfo, + ContextEntry, + Bubble, +} from '../types.js'; + +/** + * Everything that differs between the SQL engines. + * + * SQLite, Postgres and DuckDB share one set of values; SQL Server / Azure SQL + * diverges on all four, which is why these are dialect properties rather than + * constants. + */ +export interface Dialect { + name: string; + /** Render the nth (1-based) bind placeholder: `?`, `$1` or `@p1`. */ + placeholder(index: number): string; + /** Statements that create the schema if it is not already present. */ + ddl: string[]; + /** Join column expressions into a single string: `a || b` or `a + b`. */ + concat(parts: string[]): string; +} + +export interface SqlDriver { + readonly dialect: Dialect; + /** Run a statement with no bind parameters (DDL). */ + exec(sql: string): Promise; + /** Run a mutating statement. */ + run(sql: string, params: unknown[]): Promise; + /** Run a query and return its rows. */ + all(sql: string, params: unknown[]): Promise; + close(): Promise; +} + +/** + * The portable schema, used by SQLite, Postgres and DuckDB unchanged. + * + * Every column is TEXT and `tags` holds a JSON-encoded string array rather than + * a native array type, so no query below needs to branch on the engine. + */ +export const STANDARD_DDL = [ + `CREATE TABLE IF NOT EXISTS oc_bubbles ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS oc_contexts ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL, + tags TEXT NOT NULL, + source TEXT NOT NULL, + bubble_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS oc_contexts_bubble_idx ON oc_contexts (bubble_id)`, +]; + +/** + * Serialise one bootstrap step across every process opening the same database. + * + * `IF OBJECT_ID(…) IS NULL CREATE TABLE …` is a check followed by a create, and + * SQL Server takes incompatible metadata locks while running it. Two processes + * opening the same fresh database therefore do not merely both try to create the + * table — they deadlock, and SQL Server kills one of them outright. + * + * An application lock makes the check-and-create a single critical section that + * spans processes, so the second connection waits and then finds the table. The + * lock is owned by the transaction, so it is released even if the batch throws. + */ +const mssqlBootstrap = (body: string) => ` + BEGIN TRANSACTION; + BEGIN TRY + EXEC sp_getapplock @Resource = 'opencontext_schema', @LockMode = 'Exclusive', + @LockOwner = 'Transaction', @LockTimeout = 30000; + ${body}; + COMMIT TRANSACTION; + END TRY + BEGIN CATCH + IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION; + THROW; + END CATCH`; + +/** + * Every other engine compares and sorts text the same way on every install; SQL + * Server does whatever the database's collation says, and that is chosen once at + * `CREATE DATABASE` and never mentioned again. Left to the default, the same + * store would behave differently on two Azure servers: + * + * - Key columns are compared byte-for-byte, so an id matches only its own exact + * spelling and `ORDER BY created_at, id` is the ordinal order the contract + * documents. Under a default (case-insensitive) collation `getContext` would + * answer to the wrong casing, which no other backend does. + * - Searched columns get a fixed, culture-neutral collation, because `LOWER()` + * follows the collation's language. Under a Turkish or Azerbaijani collation + * `LOWER(N'Istanbul')` is `ıstanbul` with a dotless i, which never matches the + * dotted i that JavaScript's `toLowerCase` produces — every recall, tag filter + * and search over content holding a capital I silently returns nothing. + * Accent sensitivity is pinned for the same reason: `café` must not answer to + * `cafe` here when it does not anywhere else. + */ +const MSSQL_KEY_COLLATE = 'COLLATE Latin1_General_BIN2'; +const MSSQL_TEXT_COLLATE = 'COLLATE Latin1_General_CI_AS'; + +/** + * SQL Server / Azure SQL. It has no `CREATE TABLE IF NOT EXISTS`, deprecates + * `TEXT` in favour of `NVARCHAR(MAX)`, and cannot index an unbounded column — + * hence the fixed width on the key columns. + */ +export const MSSQL_DDL = [ + mssqlBootstrap(` + IF OBJECT_ID('oc_bubbles', 'U') IS NULL + CREATE TABLE oc_bubbles ( + id NVARCHAR(64) ${MSSQL_KEY_COLLATE} PRIMARY KEY, + name NVARCHAR(MAX) ${MSSQL_TEXT_COLLATE} NOT NULL, + description NVARCHAR(MAX) ${MSSQL_TEXT_COLLATE}, + created_at NVARCHAR(64) ${MSSQL_KEY_COLLATE} NOT NULL, + updated_at NVARCHAR(64) ${MSSQL_KEY_COLLATE} NOT NULL + )`), + mssqlBootstrap(` + IF OBJECT_ID('oc_contexts', 'U') IS NULL + CREATE TABLE oc_contexts ( + id NVARCHAR(64) ${MSSQL_KEY_COLLATE} PRIMARY KEY, + content NVARCHAR(MAX) ${MSSQL_TEXT_COLLATE} NOT NULL, + tags NVARCHAR(MAX) ${MSSQL_TEXT_COLLATE} NOT NULL, + source NVARCHAR(MAX) ${MSSQL_TEXT_COLLATE} NOT NULL, + bubble_id NVARCHAR(64) ${MSSQL_KEY_COLLATE}, + created_at NVARCHAR(64) ${MSSQL_KEY_COLLATE} NOT NULL, + updated_at NVARCHAR(64) ${MSSQL_KEY_COLLATE} NOT NULL + )`), + mssqlBootstrap(` + IF NOT EXISTS ( + SELECT 1 FROM sys.indexes + WHERE name = 'oc_contexts_bubble_idx' AND object_id = OBJECT_ID('oc_contexts') + ) + CREATE INDEX oc_contexts_bubble_idx ON oc_contexts (bubble_id)`), +]; + +const pipeConcat = (parts: string[]) => parts.join(" || "); + +export const QUESTION_MARK_DIALECT = (name: string): Dialect => ({ + name, + placeholder: () => '?', + ddl: STANDARD_DDL, + concat: pipeConcat, +}); + +export const NUMBERED_DIALECT = (name: string): Dialect => ({ + name, + placeholder: (index) => `$${index}`, + ddl: STANDARD_DDL, + concat: pipeConcat, +}); + +/** + * MySQL and MariaDB. + * + * `TEXT` cannot be a primary key without a prefix length, `CREATE INDEX IF NOT + * EXISTS` is not supported (so the index is declared inline), and `||` means OR + * rather than concatenation unless PIPES_AS_CONCAT is set. + * + * Every free-text column is LONGTEXT rather than TEXT because TEXT caps at 64KB + * and MySQL in strict mode rejects — rather than truncates — anything longer, so + * a `source` or bubble name that the other engines store happily would fail here. + * + * Charset and collation are pinned rather than inherited. A database created + * with a latin1 default — still the norm on pre-8.0 servers, which is exactly + * what "bring your own database" points at — cannot hold emoji or CJK at all, + * and MySQL's default `utf8mb4_0900_ai_ci` is both case- and *accent*- + * insensitive, which would make `café` match a search for `cafe` and two ids + * differing only in case collide on the primary key. `utf8mb4_bin` compares by + * code point, matching SQLite and Postgres; the case-insensitive matching the + * store contract requires comes from `LOWER()` in the queries, not from the + * collation. `utf8mb4_bin` is used in preference to `utf8mb4_0900_as_cs` + * because MariaDB has no `_0900_` collations. + */ +export const MYSQL_DDL = [ + `CREATE TABLE IF NOT EXISTS oc_bubbles ( + id VARCHAR(255) PRIMARY KEY, + name LONGTEXT NOT NULL, + description LONGTEXT, + created_at VARCHAR(64) NOT NULL, + updated_at VARCHAR(64) NOT NULL + ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin`, + `CREATE TABLE IF NOT EXISTS oc_contexts ( + id VARCHAR(255) PRIMARY KEY, + content LONGTEXT NOT NULL, + tags LONGTEXT NOT NULL, + source LONGTEXT NOT NULL, + bubble_id VARCHAR(255), + created_at VARCHAR(64) NOT NULL, + updated_at VARCHAR(64) NOT NULL, + INDEX oc_contexts_bubble_idx (bubble_id) + ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin`, +]; + +export const MYSQL_DIALECT: Dialect = { + name: 'mysql', + placeholder: () => '?', + ddl: MYSQL_DDL, + concat: (parts) => `CONCAT(${parts.join(', ')})`, +}; + +export const MSSQL_DIALECT: Dialect = { + name: 'mssql', + placeholder: (index) => `@p${index}`, + ddl: MSSQL_DDL, + concat: (parts) => parts.join(' + '), +}; + +interface ContextRow { + id: string; + content: string; + tags: string; + source: string; + bubble_id: string | null; + created_at: string; + updated_at: string; +} + +interface BubbleRow { + id: string; + name: string; + description: string | null; + created_at: string; + updated_at: string; +} + +const CONTEXT_COLUMNS = 'id, content, tags, source, bubble_id, created_at, updated_at'; +const BUBBLE_COLUMNS = 'id, name, description, created_at, updated_at'; +const CONTEXT_ORDER = 'ORDER BY created_at ASC, id ASC'; + +/** + * Rewrite `?` placeholders into the dialect's own style. + * + * Queries below are written with `?` because it reads better; Postgres gets + * `$1`, `$2`, … Literal `?` never appears inside our SQL strings otherwise. + */ +function bind(dialect: Dialect, sql: string): string { + let index = 0; + return sql.replace(/\?/g, () => dialect.placeholder(++index)); +} + +/** + * Escape LIKE wildcards so a user's `%` or `_` matches literally. + * + * `!` is the escape character rather than the more usual `\`, because MySQL + * treats backslash as an escape inside string literals too, so `ESCAPE '\'` + * has to be double-escaped there and nowhere else. `!` needs no quoting in any + * of the five engines. + */ +const LIKE_ESCAPE = "ESCAPE '!'"; + +function likeTerm(value: string): string { + return `%${value.toLowerCase().replace(/([!%_])/g, '!$1')}%`; +} + +function rowToEntry(row: ContextRow): ContextEntry { + const entry: ContextEntry = { + id: row.id, + content: row.content, + tags: JSON.parse(row.tags) as string[], + source: row.source, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; + if (row.bubble_id !== null && row.bubble_id !== undefined) { + entry.bubbleId = row.bubble_id; + } + return entry; +} + +function rowToBubble(row: BubbleRow): Bubble { + const bubble: Bubble = { + id: row.id, + name: row.name, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; + if (row.description !== null && row.description !== undefined) { + bubble.description = row.description; + } + return bubble; +} + +/** + * The whole storage contract implemented once against `SqlDriver`. + * + * Existence is checked with a SELECT before every mutation rather than reading + * an affected-row count, because the three drivers report that count in three + * incompatible ways — and one of them not at all. + */ +export function createSqlAdapter(driver: SqlDriver, info: AdapterInfo): ContextStoreAdapter { + const { dialect } = driver; + const sql = (text: string) => bind(dialect, text); + + async function findContextRow(id: string): Promise { + const rows = await driver.all( + sql(`SELECT ${CONTEXT_COLUMNS} FROM oc_contexts WHERE id = ?`), + [id], + ); + return rows[0]; + } + + async function findBubbleRow(id: string): Promise { + const rows = await driver.all( + sql(`SELECT ${BUBBLE_COLUMNS} FROM oc_bubbles WHERE id = ?`), + [id], + ); + return rows[0]; + } + + return { + info, + + async connect() { + for (const statement of dialect.ddl) { + await driver.exec(statement); + } + }, + + async close() { + await driver.close(); + }, + + async ping() { + await driver.all('SELECT 1', []); + }, + + // ----------------------------------------------------------------------- + // Contexts + // ----------------------------------------------------------------------- + + async saveContext(content, tags = [], source = 'chat', bubbleId) { + const now = new Date().toISOString(); + const entry: ContextEntry = { + id: randomUUID(), + content, + tags, + source, + createdAt: now, + updatedAt: now, + }; + if (bubbleId !== undefined) { + entry.bubbleId = bubbleId; + } + await driver.run( + sql( + `INSERT INTO oc_contexts (${CONTEXT_COLUMNS}) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ), + [entry.id, content, JSON.stringify(tags), source, bubbleId ?? null, now, now], + ); + return entry; + }, + + async recallContext(query) { + const needle = likeTerm(query); + const rows = await driver.all( + sql( + `SELECT ${CONTEXT_COLUMNS} FROM oc_contexts + WHERE LOWER(content) LIKE ? ${LIKE_ESCAPE} + OR LOWER(tags) LIKE ? ${LIKE_ESCAPE} + ${CONTEXT_ORDER}`, + ), + [needle, needle], + ); + return rows.map(rowToEntry); + }, + + async listContexts(tag) { + if (!tag) { + const rows = await driver.all( + `SELECT ${CONTEXT_COLUMNS} FROM oc_contexts ${CONTEXT_ORDER}`, + [], + ); + return rows.map(rowToEntry); + } + // Tags are a JSON array, so the surrounding quotes make this an exact + // element match rather than a prefix match: `"work"` ≠ `"workspace"`. + const needle = likeTerm(JSON.stringify(tag.toLowerCase())); + const rows = await driver.all( + sql( + `SELECT ${CONTEXT_COLUMNS} FROM oc_contexts + WHERE LOWER(tags) LIKE ? ${LIKE_ESCAPE} ${CONTEXT_ORDER}`, + ), + [needle], + ); + return rows.map(rowToEntry); + }, + + async listContextsByBubble(bubbleId) { + const rows = await driver.all( + sql( + `SELECT ${CONTEXT_COLUMNS} FROM oc_contexts WHERE bubble_id = ? ${CONTEXT_ORDER}`, + ), + [bubbleId], + ); + return rows.map(rowToEntry); + }, + + async getContext(id) { + const row = await findContextRow(id); + return row ? rowToEntry(row) : undefined; + }, + + async updateContext(id, content, tags, bubbleId) { + const existing = await findContextRow(id); + if (!existing) { + return undefined; + } + const updatedAt = new Date().toISOString(); + const nextTags = tags !== undefined ? JSON.stringify(tags) : existing.tags; + const nextBubble = + bubbleId === undefined ? existing.bubble_id : bubbleId === null ? null : bubbleId; + + await driver.run( + sql( + `UPDATE oc_contexts + SET content = ?, tags = ?, bubble_id = ?, updated_at = ? + WHERE id = ?`, + ), + [content, nextTags, nextBubble, updatedAt, id], + ); + return rowToEntry({ + ...existing, + content, + tags: nextTags, + bubble_id: nextBubble, + updated_at: updatedAt, + }); + }, + + async deleteContext(id) { + if (!(await findContextRow(id))) { + return false; + } + await driver.run(sql('DELETE FROM oc_contexts WHERE id = ?'), [id]); + return true; + }, + + async searchContexts(query) { + const terms = query.toLowerCase().split(/\s+/).filter(Boolean); + if (terms.length === 0) { + const rows = await driver.all( + `SELECT ${CONTEXT_COLUMNS} FROM oc_contexts ${CONTEXT_ORDER}`, + [], + ); + return rows.map(rowToEntry); + } + const haystack = `LOWER(${dialect.concat(['content', "' '", 'tags', "' '", 'source'])})`; + const clauses = terms.map(() => `${haystack} LIKE ? ${LIKE_ESCAPE}`).join(' AND '); + const rows = await driver.all( + sql( + `SELECT ${CONTEXT_COLUMNS} FROM oc_contexts WHERE ${clauses} ${CONTEXT_ORDER}`, + ), + terms.map(likeTerm), + ); + return rows.map(rowToEntry); + }, + + // ----------------------------------------------------------------------- + // Bubbles + // ----------------------------------------------------------------------- + + async createBubble(name, description) { + const now = new Date().toISOString(); + const bubble: Bubble = { id: randomUUID(), name, createdAt: now, updatedAt: now }; + if (description !== undefined) { + bubble.description = description; + } + await driver.run( + sql(`INSERT INTO oc_bubbles (${BUBBLE_COLUMNS}) VALUES (?, ?, ?, ?, ?)`), + [bubble.id, name, description ?? null, now, now], + ); + return bubble; + }, + + async listBubbles() { + const rows = await driver.all( + `SELECT ${BUBBLE_COLUMNS} FROM oc_bubbles ${CONTEXT_ORDER}`, + [], + ); + return rows.map(rowToBubble); + }, + + async getBubble(id) { + const row = await findBubbleRow(id); + return row ? rowToBubble(row) : undefined; + }, + + async updateBubble(id, name, description) { + const existing = await findBubbleRow(id); + if (!existing) { + return undefined; + } + const updatedAt = new Date().toISOString(); + const nextDescription = description !== undefined ? description : existing.description; + await driver.run( + sql('UPDATE oc_bubbles SET name = ?, description = ?, updated_at = ? WHERE id = ?'), + [name, nextDescription, updatedAt, id], + ); + return rowToBubble({ + ...existing, + name, + description: nextDescription, + updated_at: updatedAt, + }); + }, + + async deleteBubble(id, deleteContexts = false) { + if (!(await findBubbleRow(id))) { + return false; + } + await driver.run(sql('DELETE FROM oc_bubbles WHERE id = ?'), [id]); + if (deleteContexts) { + await driver.run(sql('DELETE FROM oc_contexts WHERE bubble_id = ?'), [id]); + } else { + await driver.run( + sql('UPDATE oc_contexts SET bubble_id = NULL WHERE bubble_id = ?'), + [id], + ); + } + return true; + }, + }; +} diff --git a/src/store/adapters/surreal.ts b/src/store/adapters/surreal.ts new file mode 100644 index 0000000..092c94c --- /dev/null +++ b/src/store/adapters/surreal.ts @@ -0,0 +1,401 @@ +import { randomUUID } from 'crypto'; +import { + type ContextStoreAdapter, + type AdapterInfo, + type ContextEntry, + type Bubble, + InvalidDsnError, +} from '../types.js'; +import type { ParsedDsn } from '../dsn.js'; +import { importOptional } from '../drivers/optional.js'; + +interface SurrealClient { + connect(endpoint: string, options: Record): Promise; + query(sql: string, vars?: Record): Promise; + close(): Promise; +} + +interface ContextRecord { + uid: string; + content: string; + tags: string[]; + source: string; + bubble_uid: string | null; + created_at: string; + updated_at: string; +} + +interface BubbleRecord { + uid: string; + name: string; + description: string | null; + created_at: string; + updated_at: string; +} + +const CONTEXT_TABLE = 'oc_context'; +const BUBBLE_TABLE = 'oc_bubble'; +const ORDER = 'ORDER BY created_at ASC, uid ASC'; + +/** Quote a namespace/database name — DEFINE statements cannot take bind variables. */ +function ident(name: string): string { + return '`' + name.replace(/`/g, '\\`') + '`'; +} + +/** + * Build the sign-in payload for the connection string's credentials. + * + * SurrealDB users are scoped, and the payload has to name the scope: a root user + * signs in with only a username and password, a namespace or database user must + * also say which namespace and database it belongs to. Sending the wrong shape + * fails as a flat "There was a problem with authentication", and nothing in the + * connection string distinguishes the two, so `?auth=` selects the level. + */ +function authFor(dsn: ParsedDsn): Record | undefined { + if (!dsn.username) { + return undefined; + } + const credentials = { username: dsn.username, password: dsn.password ?? '' }; + const level = (dsn.params.auth ?? 'root').toLowerCase(); + + if (level === 'root') { + return credentials; + } + if (level === 'namespace' || level === 'ns') { + return { namespace: dsn.namespace, ...credentials }; + } + if (level === 'database' || level === 'db') { + return { namespace: dsn.namespace, database: dsn.database, ...credentials }; + } + throw new InvalidDsnError( + `Unsupported SurrealDB auth level "${dsn.params.auth}". ` + + 'Use auth=root (the default), auth=namespace or auth=database.', + ); +} + +/** + * Denormalised lowercase fields. + * + * SurrealQL can lowercase on read, but doing it on write keeps every predicate a + * plain `string::contains` and avoids per-version differences in closure syntax. + * + * Every read still coalesces these with `?? ''`, because a row written by + * anything other than this adapter will not have them, and passing NONE to + * `string::contains` fails the whole query rather than skipping the row. + */ +function searchFields(content: string, tags: string[], source: string) { + return { + tags_lower: tags.map((tag) => tag.toLowerCase()), + tags_text: tags.join(' ').toLowerCase(), + search_text: `${content} ${tags.join(' ')} ${source}`.toLowerCase(), + }; +} + +function toEntry(record: ContextRecord): ContextEntry { + const entry: ContextEntry = { + id: record.uid, + content: record.content, + tags: record.tags ?? [], + source: record.source, + createdAt: record.created_at, + updatedAt: record.updated_at, + }; + if (record.bubble_uid !== null && record.bubble_uid !== undefined) { + entry.bubbleId = record.bubble_uid; + } + return entry; +} + +function toBubble(record: BubbleRecord): Bubble { + const bubble: Bubble = { + id: record.uid, + name: record.name, + createdAt: record.created_at, + updatedAt: record.updated_at, + }; + if (record.description !== null && record.description !== undefined) { + bubble.description = record.description; + } + return bubble; +} + +/** + * SurrealDB, embedded or remote. + * + * Records carry their own `uid` string rather than using opencontext ids as + * Surreal record ids, so nothing here depends on how a given SDK version + * serialises `RecordId`. + */ +export async function createSurrealAdapter( + dsn: ParsedDsn, + info: AdapterInfo, +): Promise { + const { Surreal } = await importOptional<{ Surreal: new () => SurrealClient }>( + 'surrealdb', + 'surrealdb', + ); + + const db = new Surreal(); + + /** SurrealDB returns one result block per statement; we always send one. */ + async function q(sql: string, vars: Record = {}): Promise { + const result = await db.query(sql, vars); + return (result[0] ?? []) as T[]; + } + + /** Run setup DDL that a correctly provisioned but unprivileged user may refuse. */ + async function bestEffort(sql: string): Promise { + try { + await db.query(sql); + } catch { + // Deliberately ignored — see the call site. + } + } + + async function findContext(id: string): Promise { + const rows = await q( + `SELECT * FROM ${CONTEXT_TABLE} WHERE uid = $uid LIMIT 1`, + { uid: id }, + ); + return rows[0]; + } + + async function findBubble(id: string): Promise { + const rows = await q( + `SELECT * FROM ${BUBBLE_TABLE} WHERE uid = $uid LIMIT 1`, + { uid: id }, + ); + return rows[0]; + } + + return { + info, + + async connect() { + const options: Record = { + namespace: dsn.namespace, + database: dsn.database, + }; + const authentication = authFor(dsn); + if (authentication) { + options.authentication = authentication; + } + await db.connect(dsn.endpoint!, options); + + // A namespace and database are no longer created implicitly by selecting + // them, so create them here. Only a root user may: a database-scoped user + // is refused, and does not need it, because its database already exists. + // Swallowing that refusal is safe — if the database really is missing, the + // DEFINE TABLE below fails and reports it. + await bestEffort(`DEFINE NAMESPACE IF NOT EXISTS ${ident(dsn.namespace!)}`); + await bestEffort(`DEFINE DATABASE IF NOT EXISTS ${ident(dsn.database!)}`); + + // SELECT, UPDATE and DELETE all error on a table that was never defined, + // so both tables have to exist before the first read, not the first write. + await db.query(`DEFINE TABLE IF NOT EXISTS ${CONTEXT_TABLE} SCHEMALESS`); + await db.query(`DEFINE TABLE IF NOT EXISTS ${BUBBLE_TABLE} SCHEMALESS`); + + await db.query(`DEFINE INDEX IF NOT EXISTS ${CONTEXT_TABLE}_uid + ON ${CONTEXT_TABLE} FIELDS uid UNIQUE`); + await db.query(`DEFINE INDEX IF NOT EXISTS ${BUBBLE_TABLE}_uid + ON ${BUBBLE_TABLE} FIELDS uid UNIQUE`); + await db.query(`DEFINE INDEX IF NOT EXISTS ${CONTEXT_TABLE}_bubble + ON ${CONTEXT_TABLE} FIELDS bubble_uid`); + }, + + async close() { + await db.close(); + }, + + async ping() { + await db.query('RETURN 1'); + }, + + // ----------------------------------------------------------------------- + // Contexts + // ----------------------------------------------------------------------- + + async saveContext(content, tags = [], source = 'chat', bubbleId) { + const now = new Date().toISOString(); + const uid = randomUUID(); + await q(`CREATE ${CONTEXT_TABLE} CONTENT $data`, { + data: { + uid, + content, + tags, + source, + bubble_uid: bubbleId ?? null, + created_at: now, + updated_at: now, + ...searchFields(content, tags, source), + }, + }); + const entry: ContextEntry = { + id: uid, + content, + tags, + source, + createdAt: now, + updatedAt: now, + }; + if (bubbleId !== undefined) { + entry.bubbleId = bubbleId; + } + return entry; + }, + + async recallContext(query) { + const needle = query.toLowerCase(); + const rows = await q( + `SELECT * FROM ${CONTEXT_TABLE} + WHERE string::contains(string::lowercase(content ?? ''), $needle) + OR string::contains(tags_text ?? '', $needle) + ${ORDER}`, + { needle }, + ); + return rows.map(toEntry); + }, + + async listContexts(tag) { + if (!tag) { + return (await q(`SELECT * FROM ${CONTEXT_TABLE} ${ORDER}`)).map(toEntry); + } + const rows = await q( + `SELECT * FROM ${CONTEXT_TABLE} WHERE $tag IN tags_lower ${ORDER}`, + { tag: tag.toLowerCase() }, + ); + return rows.map(toEntry); + }, + + async listContextsByBubble(bubbleId) { + const rows = await q( + `SELECT * FROM ${CONTEXT_TABLE} WHERE bubble_uid = $bubble ${ORDER}`, + { bubble: bubbleId }, + ); + return rows.map(toEntry); + }, + + async getContext(id) { + const record = await findContext(id); + return record ? toEntry(record) : undefined; + }, + + async updateContext(id, content, tags, bubbleId) { + const existing = await findContext(id); + if (!existing) { + return undefined; + } + const nextTags = tags !== undefined ? tags : (existing.tags ?? []); + const nextBubble = + bubbleId === undefined ? existing.bubble_uid : bubbleId === null ? null : bubbleId; + const updatedAt = new Date().toISOString(); + + await q( + `UPDATE ${CONTEXT_TABLE} MERGE $data WHERE uid = $uid`, + { + uid: id, + data: { + content, + tags: nextTags, + bubble_uid: nextBubble, + updated_at: updatedAt, + ...searchFields(content, nextTags, existing.source), + }, + }, + ); + return toEntry({ + ...existing, + content, + tags: nextTags, + bubble_uid: nextBubble, + updated_at: updatedAt, + }); + }, + + async deleteContext(id) { + if (!(await findContext(id))) { + return false; + } + await q(`DELETE ${CONTEXT_TABLE} WHERE uid = $uid`, { uid: id }); + return true; + }, + + async searchContexts(query) { + const terms = query.toLowerCase().split(/\s+/).filter(Boolean); + if (terms.length === 0) { + return (await q(`SELECT * FROM ${CONTEXT_TABLE} ${ORDER}`)).map(toEntry); + } + const vars: Record = {}; + const clauses = terms.map((term, index) => { + vars[`t${index}`] = term; + return `string::contains(search_text ?? '', $t${index})`; + }); + const rows = await q( + `SELECT * FROM ${CONTEXT_TABLE} WHERE ${clauses.join(' AND ')} ${ORDER}`, + vars, + ); + return rows.map(toEntry); + }, + + // ----------------------------------------------------------------------- + // Bubbles + // ----------------------------------------------------------------------- + + async createBubble(name, description) { + const now = new Date().toISOString(); + const uid = randomUUID(); + await q(`CREATE ${BUBBLE_TABLE} CONTENT $data`, { + data: { + uid, + name, + description: description ?? null, + created_at: now, + updated_at: now, + }, + }); + const bubble: Bubble = { id: uid, name, createdAt: now, updatedAt: now }; + if (description !== undefined) { + bubble.description = description; + } + return bubble; + }, + + async listBubbles() { + return (await q(`SELECT * FROM ${BUBBLE_TABLE} ${ORDER}`)).map(toBubble); + }, + + async getBubble(id) { + const record = await findBubble(id); + return record ? toBubble(record) : undefined; + }, + + async updateBubble(id, name, description) { + const existing = await findBubble(id); + if (!existing) { + return undefined; + } + const nextDescription = description !== undefined ? description : existing.description; + const updatedAt = new Date().toISOString(); + await q(`UPDATE ${BUBBLE_TABLE} MERGE $data WHERE uid = $uid`, { + uid: id, + data: { name, description: nextDescription, updated_at: updatedAt }, + }); + return toBubble({ ...existing, name, description: nextDescription, updated_at: updatedAt }); + }, + + async deleteBubble(id, deleteContexts = false) { + if (!(await findBubble(id))) { + return false; + } + await q(`DELETE ${BUBBLE_TABLE} WHERE uid = $uid`, { uid: id }); + if (deleteContexts) { + await q(`DELETE ${CONTEXT_TABLE} WHERE bubble_uid = $bubble`, { bubble: id }); + } else { + await q(`UPDATE ${CONTEXT_TABLE} SET bubble_uid = NONE WHERE bubble_uid = $bubble`, { + bubble: id, + }); + } + return true; + }, + }; +} diff --git a/src/store/config.ts b/src/store/config.ts new file mode 100644 index 0000000..cb955cb --- /dev/null +++ b/src/store/config.ts @@ -0,0 +1,104 @@ +import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync } from 'fs'; +import { dirname, join } from 'path'; +import { homedir } from 'os'; +import { redactDsn } from './dsn.js'; + +const CONFIG_VERSION = 1; + +export interface StoredConfig { + version: number; + database?: { url: string }; +} + +export function getConfigPath(): string { + return process.env.OPENCONTEXT_CONFIG_PATH ?? join(homedir(), '.opencontext', 'config.json'); +} + +export function getDefaultJsonPath(): string { + return join(homedir(), '.opencontext', 'contexts.json'); +} + +export function readConfig(): StoredConfig { + const path = getConfigPath(); + if (!existsSync(path)) { + return { version: CONFIG_VERSION }; + } + try { + return JSON.parse(readFileSync(path, 'utf-8')) as StoredConfig; + } catch { + // A corrupt config must not make opencontext unusable — fall back to the + // default store and let the user fix or overwrite it from the settings page. + return { version: CONFIG_VERSION }; + } +} + +/** + * Persist the connection string. + * + * The file is written with mode 0600 because a connection string routinely + * carries a database password. + */ +export function writeDatabaseUrl(url: string): void { + const path = getConfigPath(); + const directory = dirname(path); + if (!existsSync(directory)) { + mkdirSync(directory, { recursive: true, mode: 0o700 }); + } + const next: StoredConfig = { ...readConfig(), version: CONFIG_VERSION, database: { url } }; + writeFileSync(path, JSON.stringify(next, null, 2) + '\n', { encoding: 'utf-8', mode: 0o600 }); + chmodSync(path, 0o600); +} + +export function clearDatabaseUrl(): void { + const path = getConfigPath(); + if (!existsSync(path)) { + return; + } + const next = readConfig(); + delete next.database; + writeFileSync(path, JSON.stringify(next, null, 2) + '\n', { encoding: 'utf-8', mode: 0o600 }); +} + +export type ConfigSource = 'env' | 'config-file' | 'legacy-store-path' | 'default'; + +export interface ResolvedDatabase { + url: string; + redacted: string; + source: ConfigSource; + /** True when the value came from the environment and the UI cannot change it. */ + locked: boolean; +} + +/** + * Work out which store to open. + * + * Environment wins over the saved config so a container can override whatever a + * user saved from the settings page, and the legacy `OPENCONTEXT_STORE_PATH` + * keeps working by mapping onto the JSON adapter — an existing install with no + * configuration resolves to exactly the file it has always used. + */ +export function resolveDatabase(): ResolvedDatabase { + const fromEnv = process.env.OPENCONTEXT_DB_URL?.trim(); + if (fromEnv) { + return { url: fromEnv, redacted: redactDsn(fromEnv), source: 'env', locked: true }; + } + + const fromConfig = readConfig().database?.url?.trim(); + if (fromConfig) { + return { + url: fromConfig, + redacted: redactDsn(fromConfig), + source: 'config-file', + locked: false, + }; + } + + const legacyPath = process.env.OPENCONTEXT_STORE_PATH?.trim(); + if (legacyPath) { + const url = `json://${legacyPath}`; + return { url, redacted: url, source: 'legacy-store-path', locked: true }; + } + + const url = `json://${getDefaultJsonPath()}`; + return { url, redacted: url, source: 'default', locked: false }; +} diff --git a/src/store/drivers/d1.ts b/src/store/drivers/d1.ts new file mode 100644 index 0000000..da81dfc --- /dev/null +++ b/src/store/drivers/d1.ts @@ -0,0 +1,63 @@ +import { QUESTION_MARK_DIALECT, type SqlDriver } from '../adapters/sql.js'; +import { InvalidDsnError } from '../types.js'; +import type { ParsedDsn } from '../dsn.js'; + +interface D1Response { + success: boolean; + errors?: { message: string }[]; + result?: { results?: unknown[] }[]; +} + +/** + * Cloudflare D1 over its HTTP API. + * + * D1 is SQLite, so it shares the standard dialect. This driver talks to the REST + * endpoint with `fetch`, which means it needs no dependency at all — the one + * remote backend with nothing to install. + */ +export async function createD1Driver(dsn: ParsedDsn): Promise { + const token = dsn.params.apiToken ?? process.env.CLOUDFLARE_API_TOKEN; + if (!token) { + throw new InvalidDsnError( + 'Cloudflare D1 needs an API token. Pass it as ?apiToken=… or set CLOUDFLARE_API_TOKEN.', + ); + } + + const endpoint = + `https://api.cloudflare.com/client/v4/accounts/${dsn.accountId}` + + `/d1/database/${dsn.databaseId}/query`; + + async function send(sql: string, params: unknown[]): Promise { + const response = await fetch(endpoint, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ sql, params }), + }); + + const body = (await response.json()) as D1Response; + if (!response.ok || !body.success) { + const detail = body.errors?.map((e) => e.message).join('; ') ?? response.statusText; + throw new Error(`Cloudflare D1 query failed: ${detail}`); + } + return body.result?.[0]?.results ?? []; + } + + return { + dialect: QUESTION_MARK_DIALECT('d1'), + async exec(sql) { + await send(sql, []); + }, + async run(sql, params) { + await send(sql, params); + }, + async all(sql: string, params: unknown[]) { + return (await send(sql, params)) as T[]; + }, + async close() { + // Stateless HTTP — nothing to release. + }, + }; +} diff --git a/src/store/drivers/duckdb.ts b/src/store/drivers/duckdb.ts new file mode 100644 index 0000000..021d045 --- /dev/null +++ b/src/store/drivers/duckdb.ts @@ -0,0 +1,74 @@ +import { existsSync, mkdirSync } from 'fs'; +import { dirname } from 'path'; +import { QUESTION_MARK_DIALECT, type SqlDriver } from '../adapters/sql.js'; +import { importOptional } from './optional.js'; +import type { ParsedDsn } from '../dsn.js'; + +/** + * The slice of `@duckdb/node-api` this driver uses, verified against 1.5.5-r.4. + * + * `run` and `runAndReadAll` both take the bind values as their second argument + * and resolve once the statement has finished, so no separate prepare step is + * needed. `getRowObjects` hands back plain JS values — every column in the + * schema is TEXT, which DuckDB returns as a JS string and SQL NULL as `null`. + */ +interface DuckDbConnection { + run(sql: string, values?: unknown[]): Promise; + runAndReadAll(sql: string, values?: unknown[]): Promise<{ getRowObjects(): unknown[] }>; + closeSync(): void; +} + +interface DuckDbInstance { + connect(): Promise; + closeSync(): void; +} + +/** + * DuckDB — embedded, column-oriented, and the right pick when the context store + * is large enough that people want to run analytical queries over it directly. + * + * Uses the same SQL as SQLite and Postgres, so it shares the standard dialect: + * `?` placeholders, `CREATE TABLE/INDEX IF NOT EXISTS` and `||` concatenation + * all behave as that dialect expects. + */ +export async function createDuckDbDriver(dsn: ParsedDsn): Promise { + const { DuckDBInstance } = await importOptional<{ + DuckDBInstance: { create(path: string): Promise }; + }>('@duckdb/node-api', 'duckdb'); + + const path = dsn.path!; + if (path !== ':memory:') { + const directory = dirname(path); + if (directory && !existsSync(directory)) { + mkdirSync(directory, { recursive: true }); + } + } + + const instance = await DuckDBInstance.create(path); + const connection = await instance.connect(); + + return { + dialect: QUESTION_MARK_DIALECT('duckdb'), + + async exec(sql) { + await connection.run(sql); + }, + + async run(sql, params) { + await connection.run(sql, params); + }, + + async all(sql: string, params: unknown[]) { + const reader = await connection.runAndReadAll(sql, params); + return reader.getRowObjects() as T[]; + }, + + async close() { + // The instance owns the database handle; closing only the connection + // leaves it — and the several megabytes of buffer pool behind it — alive + // for the lifetime of the process. Both have to go. + connection.closeSync(); + instance.closeSync(); + }, + }; +} diff --git a/src/store/drivers/dynamodb.ts b/src/store/drivers/dynamodb.ts new file mode 100644 index 0000000..04b1062 --- /dev/null +++ b/src/store/drivers/dynamodb.ts @@ -0,0 +1,212 @@ +import type { Collection, Document, DocumentDriver } from '../adapters/document.js'; +import type { ParsedDsn } from '../dsn.js'; +import { importOptional } from './optional.js'; + +/** + * Contexts and bubbles share one table, split by partition key, so listing + * either collection is a `Query` against one partition rather than a full-table + * `Scan`. + */ +const PARTITION: Record = { + contexts: 'CONTEXT', + bubbles: 'BUBBLE', +}; + +/** How long to wait for a newly created table to accept reads and writes. */ +const ACTIVE_POLL_ATTEMPTS = 60; +const ACTIVE_POLL_INTERVAL_MS = 1000; + +type Sender = { send(command: unknown): Promise> }; + +function isAwsError(error: unknown, name: string): boolean { + return (error as { name?: string } | undefined)?.name === name; +} + +/** + * Amazon DynamoDB. + * + * The table is created on first connect if it is absent, matching how the + * file-based backends create their store on first use. + * + * Every read is strongly consistent. DynamoDB reads are eventually consistent by + * default, which would let `getContext` right after `saveContext` return nothing + * — the store contract (and the conformance suite) requires read-your-writes. + * The cost is that a read consumes twice the read capacity units. + */ +export async function createDynamoDbDriver(dsn: ParsedDsn): Promise { + const client = await importOptional>( + '@aws-sdk/client-dynamodb', + 'dynamodb', + '@aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb', + ); + const lib = await importOptional>( + '@aws-sdk/lib-dynamodb', + 'dynamodb', + '@aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb', + ); + + const DynamoDBClient = client.DynamoDBClient as new (config: unknown) => Sender & { + destroy?(): void; + }; + const CreateTableCommand = client.CreateTableCommand as new (input: unknown) => unknown; + const DescribeTableCommand = client.DescribeTableCommand as new (input: unknown) => unknown; + const DynamoDBDocumentClient = lib.DynamoDBDocumentClient as { + from(base: unknown, translateConfig?: unknown): Sender; + }; + const PutCommand = lib.PutCommand as new (input: unknown) => unknown; + const GetCommand = lib.GetCommand as new (input: unknown) => unknown; + const DeleteCommand = lib.DeleteCommand as new (input: unknown) => unknown; + const QueryCommand = lib.QueryCommand as new (input: unknown) => unknown; + + const TableName = dsn.table!; + const config: Record = { region: dsn.region }; + if (dsn.params.endpoint) { + config.endpoint = dsn.params.endpoint; + } + if (dsn.params.accessKeyId && dsn.params.secretAccessKey) { + config.credentials = { + accessKeyId: dsn.params.accessKeyId, + secretAccessKey: dsn.params.secretAccessKey, + // Temporary credentials from STS or a role need the session token too. + ...(dsn.params.sessionToken ? { sessionToken: dsn.params.sessionToken } : {}), + }; + } + + const base = new DynamoDBClient(config); + const documents = DynamoDBDocumentClient.from(base, { + marshallOptions: { + // An unset optional field (`bubbleId`, `description`) is absent, not null. + removeUndefinedValues: true, + // Left off deliberately: it would rewrite empty strings and empty lists as + // NULL, so `tags: []` and empty content would not survive a round trip. + convertEmptyValues: false, + }, + }); + + /** The table's description, or undefined when it does not exist yet. */ + async function describeTable(): Promise<{ TableStatus?: string } | undefined> { + try { + const response = await base.send(new DescribeTableCommand({ TableName })); + return (response.Table as { TableStatus?: string } | undefined) ?? {}; + } catch (error) { + if (isAwsError(error, 'ResourceNotFoundException')) { + return undefined; + } + throw error; + } + } + + /** + * Wait until the table accepts reads and writes. + * + * A table that exists is not necessarily usable: while it is `CREATING`, every + * data-plane call fails with `ResourceNotFoundException`. Anything other than + * a missing table — bad credentials, wrong endpoint — is raised immediately + * rather than retried for a minute behind a misleading timeout message. + */ + async function waitUntilActive(): Promise { + for (let attempt = 0; attempt < ACTIVE_POLL_ATTEMPTS; attempt += 1) { + const table = await describeTable(); + if (table?.TableStatus === 'ACTIVE') { + return; + } + await new Promise((resolve) => setTimeout(resolve, ACTIVE_POLL_INTERVAL_MS)); + } + throw new Error(`DynamoDB table "${TableName}" did not become ACTIVE in time.`); + } + + return { + async connect() { + if (!(await describeTable())) { + try { + await base.send( + new CreateTableCommand({ + TableName, + BillingMode: 'PAY_PER_REQUEST', + AttributeDefinitions: [ + { AttributeName: 'pk', AttributeType: 'S' }, + { AttributeName: 'sk', AttributeType: 'S' }, + ], + KeySchema: [ + { AttributeName: 'pk', KeyType: 'HASH' }, + { AttributeName: 'sk', KeyType: 'RANGE' }, + ], + }), + ); + } catch (error) { + // Another process created the table between the describe and the + // create — the outcome we wanted, reported as a conflict. + if (!isAwsError(error, 'ResourceInUseException')) { + throw error; + } + } + } + // Table creation is asynchronous, and a table someone else is creating + // right now is equally unusable, so always wait rather than only after a + // create this process issued. + await waitUntilActive(); + }, + + async close() { + base.destroy?.(); + }, + + async ping() { + await base.send(new DescribeTableCommand({ TableName })); + }, + + async get(collection, id) { + const result = await documents.send( + new GetCommand({ + TableName, + Key: { pk: PARTITION[collection], sk: id }, + ConsistentRead: true, + }), + ); + const item = result.Item as Document | undefined; + if (!item) { + return undefined; + } + const { pk: _pk, sk: _sk, ...rest } = item; + return { ...rest, id }; + }, + + async put(collection, id, document) { + await documents.send( + new PutCommand({ + TableName, + Item: { ...document, pk: PARTITION[collection], sk: id }, + }), + ); + }, + + async remove(collection, id) { + await documents.send( + new DeleteCommand({ TableName, Key: { pk: PARTITION[collection], sk: id } }), + ); + }, + + async list(collection) { + const items: Document[] = []; + let startKey: unknown; + do { + const result = await documents.send( + new QueryCommand({ + TableName, + KeyConditionExpression: '#pk = :pk', + ExpressionAttributeNames: { '#pk': 'pk' }, + ExpressionAttributeValues: { ':pk': PARTITION[collection] }, + ConsistentRead: true, + ...(startKey ? { ExclusiveStartKey: startKey } : {}), + }), + ); + for (const raw of (result.Items ?? []) as Document[]) { + const { pk: _pk, sk, ...rest } = raw; + items.push({ ...rest, id: sk as string }); + } + startKey = result.LastEvaluatedKey; + } while (startKey); + return items; + }, + }; +} diff --git a/src/store/drivers/firestore.ts b/src/store/drivers/firestore.ts new file mode 100644 index 0000000..c1ee545 --- /dev/null +++ b/src/store/drivers/firestore.ts @@ -0,0 +1,78 @@ +import type { Collection, Document, DocumentDriver } from '../adapters/document.js'; +import type { ParsedDsn } from '../dsn.js'; +import { importOptional } from './optional.js'; + +const COLLECTIONS: Record = { + contexts: 'oc_contexts', + bubbles: 'oc_bubbles', +}; + +interface FirestoreDoc { + set(data: Document): Promise; + get(): Promise<{ exists: boolean; data(): Document | undefined }>; + delete(): Promise; +} + +interface FirestoreCollection { + doc(id: string): FirestoreDoc; + get(): Promise<{ docs: { id: string; data(): Document }[] }>; +} + +interface FirestoreClient { + collection(name: string): FirestoreCollection; + terminate(): Promise; +} + +/** + * Google Cloud Firestore. + * + * Credentials come from the ambient Google application-default chain, the same + * way every other Google client library resolves them, so nothing sensitive + * needs to live in the connection string. + */ +export async function createFirestoreDriver(dsn: ParsedDsn): Promise { + const { Firestore } = await importOptional<{ + Firestore: new (config: Record) => FirestoreClient; + }>('@google-cloud/firestore', 'firestore', '@google-cloud/firestore'); + + const config: Record = { projectId: dsn.project }; + if (dsn.database && dsn.database !== '(default)') { + config.databaseId = dsn.database; + } + const db = new Firestore(config); + + return { + async connect() { + // Firestore connects lazily on first operation; nothing to open here. + }, + + async close() { + await db.terminate(); + }, + + async ping() { + await db.collection(COLLECTIONS.contexts).get(); + }, + + async get(collection, id) { + const snapshot = await db.collection(COLLECTIONS[collection]).doc(id).get(); + if (!snapshot.exists) { + return undefined; + } + return { ...(snapshot.data() ?? {}), id }; + }, + + async put(collection, id, document) { + await db.collection(COLLECTIONS[collection]).doc(id).set(document); + }, + + async remove(collection, id) { + await db.collection(COLLECTIONS[collection]).doc(id).delete(); + }, + + async list(collection) { + const snapshot = await db.collection(COLLECTIONS[collection]).get(); + return snapshot.docs.map((doc) => ({ ...doc.data(), id: doc.id })); + }, + }; +} diff --git a/src/store/drivers/memory.ts b/src/store/drivers/memory.ts new file mode 100644 index 0000000..0e675c0 --- /dev/null +++ b/src/store/drivers/memory.ts @@ -0,0 +1,60 @@ +import type { Collection, Document, DocumentDriver } from '../adapters/document.js'; + +type Tables = Record>; + +/** + * Stores live for the lifetime of the process, keyed by name. + * + * Without this, reopening `memory://` would hand back an empty store and the + * HTTP server would appear to lose data every time it reconnected. Ephemeral + * means "gone when the process exits", not "gone when you look away". + */ +const stores = new Map(); + +function tablesFor(name: string): Tables { + let tables = stores.get(name); + if (!tables) { + tables = { contexts: new Map(), bubbles: new Map() }; + stores.set(name, tables); + } + return tables; +} + +/** Drop a named store. Used by tests to get a clean slate. */ +export function resetMemoryStore(name = 'default'): void { + stores.delete(name); +} + +/** + * Process-local, ephemeral storage. + * + * Useful for trying opencontext out without writing anything to disk, and it + * doubles as the reference implementation of `DocumentDriver` — the shared + * document adapter is conformance-tested through it. + */ +export function createMemoryDriver(name = 'default'): DocumentDriver { + const tables = tablesFor(name); + + return { + async connect() {}, + async close() {}, + async ping() {}, + + async get(collection, id) { + const found = tables[collection].get(id); + return found ? { ...found } : undefined; + }, + + async put(collection, id, document) { + tables[collection].set(id, { ...document }); + }, + + async remove(collection, id) { + tables[collection].delete(id); + }, + + async list(collection) { + return [...tables[collection].values()].map((document) => ({ ...document })); + }, + }; +} diff --git a/src/store/drivers/mongodb.ts b/src/store/drivers/mongodb.ts new file mode 100644 index 0000000..ac9b88d --- /dev/null +++ b/src/store/drivers/mongodb.ts @@ -0,0 +1,113 @@ +import type { Collection, Document, DocumentDriver } from '../adapters/document.js'; +import type { ParsedDsn } from '../dsn.js'; +import { importOptional } from './optional.js'; + +const COLLECTIONS: Record = { + contexts: 'oc_contexts', + bubbles: 'oc_bubbles', +}; + +interface MongoCollection { + findOne(filter: Document): Promise; + replaceOne(filter: Document, doc: Document, options: Document): Promise; + deleteOne(filter: Document): Promise; + find(filter: Document): { toArray(): Promise }; + createIndex(spec: Document, options?: Document): Promise; +} + +interface MongoDb { + collection(name: string): MongoCollection; + command(command: Document): Promise; +} + +interface MongoClientLike { + connect(): Promise; + db(name?: string): MongoDb; + close(): Promise; +} + +/** + * Rewrite the scheme to one the Node driver accepts. + * + * The connection string is handed to the driver verbatim so that every Mongo + * option keeps working, but the driver accepts only `mongodb://` and + * `mongodb+srv://` — literally, and case-sensitively. Our DSN parser is more + * forgiving: it accepts the `mongo://` alias and any casing. Without this, + * `mongo://host/db` parses fine and then dies inside the driver with + * "Invalid scheme", which reads like a bug in the user's connection string. + * + * `mongodb+srv://` is preserved, because dropping the `+srv` would turn an Atlas + * SRV lookup into a direct connection to a host that does not answer. + */ +function canonicalConnectionString(raw: string): string { + return raw.replace(/^[a-zA-Z][a-zA-Z0-9+.-]*:/, (scheme) => + scheme.toLowerCase() === 'mongodb+srv:' ? 'mongodb+srv:' : 'mongodb:', + ); +} + +/** + * MongoDB, including Atlas (`mongodb+srv://`) and Azure Cosmos DB's Mongo API. + * + * Documents are keyed by `_id` set to the opencontext id, so a lookup is a + * primary-key hit rather than a scan. + * + * One Mongo-specific limit leaks through: a single BSON document cannot exceed + * 16 MB, so a context whose content approaches that size cannot be saved here. + * Every other backend takes it. + */ +export async function createMongoDriver(dsn: ParsedDsn): Promise { + const { MongoClient } = await importOptional<{ + MongoClient: new (url: string, options?: Document) => MongoClientLike; + }>('mongodb', 'mongodb'); + + // `ignoreUndefined` keeps an absent optional field — bubbleId, description — + // absent. BSON's default is to encode `undefined` as `null`, which would make + // this the one backend that answers `null` where the others answer nothing. + const client = new MongoClient(canonicalConnectionString(dsn.raw), { + ignoreUndefined: true, + }); + let db: MongoDb; + + const collection = (name: Collection) => db.collection(COLLECTIONS[name]); + + /** Mongo stores the key as `_id`; the rest of the system calls it `id`. */ + const fromMongo = (doc: Document): Document => { + const { _id, ...rest } = doc; + return { ...rest, id: _id as string }; + }; + + return { + async connect() { + await client.connect(); + db = client.db(dsn.database); + await collection('contexts').createIndex({ bubbleId: 1 }); + await collection('contexts').createIndex({ createdAt: 1 }); + }, + + async close() { + await client.close(); + }, + + async ping() { + await db.command({ ping: 1 }); + }, + + async get(name, id) { + const found = await collection(name).findOne({ _id: id }); + return found ? fromMongo(found) : undefined; + }, + + async put(name, id, document) { + const { id: _ignored, ...rest } = document; + await collection(name).replaceOne({ _id: id }, { _id: id, ...rest }, { upsert: true }); + }, + + async remove(name, id) { + await collection(name).deleteOne({ _id: id }); + }, + + async list(name) { + return (await collection(name).find({}).toArray()).map(fromMongo); + }, + }; +} diff --git a/src/store/drivers/mssql.ts b/src/store/drivers/mssql.ts new file mode 100644 index 0000000..0a243cc --- /dev/null +++ b/src/store/drivers/mssql.ts @@ -0,0 +1,107 @@ +import { MSSQL_DIALECT, type SqlDriver } from '../adapters/sql.js'; +import { importOptional } from './optional.js'; +import type { ParsedDsn } from '../dsn.js'; + +interface MssqlRequest { + input(name: string, value: unknown): MssqlRequest; + query(sql: string): Promise<{ recordset: unknown[] }>; + batch(sql: string): Promise; +} + +interface MssqlPool { + request(): MssqlRequest; + connect(): Promise; + close(): Promise; +} + +interface MssqlModule { + ConnectionPool: new (config: unknown) => MssqlPool; +} + +/** + * SQL Server error numbers meaning "the thing this statement creates is already + * there": 2714 for a table, 1913 for an index. + * + * `IF OBJECT_ID(…) IS NULL CREATE TABLE …` is a check followed by a create, not + * one atomic step, so two processes opening the same fresh database at the same + * moment can both find nothing and both try to create it. One wins, the other + * gets these. The loser's intent is already satisfied, so it is not an error. + */ +const ALREADY_EXISTS = new Set([2714, 1913]); + +function isAlreadyExists(error: unknown): boolean { + return ALREADY_EXISTS.has((error as { number?: number }).number ?? -1); +} + +/** + * SQL Server, including Azure SQL Database. + * + * Azure requires TLS, so `encrypt` defaults to on here — the opposite of the + * driver's own default, and the setting people most often get wrong. + * + * Note that Azure SQL usernames frequently contain `@` (`admin@myserver`); those + * must be percent-encoded in the connection string or the URL parser reads the + * `@` as the credential separator. + */ +export async function createMssqlDriver(dsn: ParsedDsn): Promise { + const mssql = await importOptional('mssql', 'mssql'); + + const isAzure = (dsn.host ?? '').endsWith('.database.windows.net'); + const encrypt = dsn.params.encrypt ? dsn.params.encrypt !== 'false' : true; + + // A dedicated pool, never `mssql.connect()` — that helper caches one global + // pool per process, so a second store would silently reuse the first one's + // server and database, and the first `close()` would disconnect them all. + const pool = new mssql.ConnectionPool({ + server: dsn.host, + port: dsn.port, + user: dsn.username, + password: dsn.password, + database: dsn.database, + options: { + encrypt, + // Self-signed certificates are normal for local SQL Server and never for + // Azure, so trust follows the host rather than being a flag people forget. + trustServerCertificate: dsn.params.trustServerCertificate + ? dsn.params.trustServerCertificate !== 'false' + : !isAzure, + }, + }); + + await pool.connect(); + + function requestWith(params: unknown[]): MssqlRequest { + const request = pool.request(); + params.forEach((value, index) => request.input(`p${index + 1}`, value)); + return request; + } + + return { + dialect: MSSQL_DIALECT, + + async exec(sql) { + // `batch` rather than `query`, because the schema DDL uses `IF NOT EXISTS` + // control flow that SQL Server rejects inside a parameterised statement. + try { + await pool.request().batch(sql); + } catch (error) { + if (!isAlreadyExists(error)) { + throw error; + } + } + }, + + async run(sql, params) { + await requestWith(params).query(sql); + }, + + async all(sql: string, params: unknown[]) { + const result = await requestWith(params).query(sql); + return result.recordset as T[]; + }, + + async close() { + await pool.close(); + }, + }; +} diff --git a/src/store/drivers/mysql.ts b/src/store/drivers/mysql.ts new file mode 100644 index 0000000..385e113 --- /dev/null +++ b/src/store/drivers/mysql.ts @@ -0,0 +1,97 @@ +import { MYSQL_DIALECT, type SqlDriver } from '../adapters/sql.js'; +import { importOptional } from './optional.js'; +import type { ParsedDsn } from '../dsn.js'; + +interface MysqlPool { + query(sql: string, params?: unknown[]): Promise<[unknown, unknown]>; + execute(sql: string, params?: unknown[]): Promise<[unknown, unknown]>; + end(): Promise; +} + +type SslOption = undefined | { rejectUnauthorized: boolean }; + +/** + * Translate a TLS request into what `mysql2` expects. + * + * Both spellings are accepted: `sslmode`, which is what Postgres users type and + * what the sibling driver takes, and MySQL's own `ssl-mode` vocabulary. As with + * Postgres, `require` encrypts without verifying, because managed MySQL — RDS, + * Cloud SQL, Azure — presents a chain Node does not trust out of the box; + * verification is opt-in and, unlike before, is actually honoured when asked for. + */ +function sslOptionFor(dsn: ParsedDsn): SslOption { + const mode = (dsn.params.sslmode ?? dsn.params['ssl-mode'])?.toLowerCase(); + switch (mode) { + case 'disable': + case 'disabled': + return undefined; + case 'require': + case 'required': + case 'prefer': + case 'preferred': + case 'allow': + return { rejectUnauthorized: false }; + case 'verify-ca': + case 'verify_ca': + case 'verify-full': + case 'verify-identity': + case 'verify_identity': + return { rejectUnauthorized: true }; + default: + // `?ssl=true` predates the modes and stays an alias for unverified TLS. + return dsn.params.ssl === 'true' ? { rejectUnauthorized: false } : undefined; + } +} + +/** + * MySQL and MariaDB, which also covers PlanetScale, Azure Database for MySQL, + * Cloud SQL for MySQL, and Aurora MySQL — they all speak the same wire protocol. + */ +export async function createMysqlDriver(dsn: ParsedDsn): Promise { + const mysql = await importOptional<{ createPool(config: unknown): MysqlPool }>( + 'mysql2/promise', + 'mysql', + 'mysql2', + ); + + const config: Record = { + host: dsn.host, + port: dsn.port, + user: dsn.username, + password: dsn.password, + database: dsn.database, + // Without this, `?` inside a string literal can be mistaken for a parameter. + namedPlaceholders: false, + // Pinned to match the utf8mb4 schema. The driver already defaults to + // utf8mb4, but an implicit default is a poor thing to rest emoji on. + charset: 'utf8mb4', + }; + const ssl = sslOptionFor(dsn); + if (ssl !== undefined) { + config.ssl = ssl; + } + + const pool = mysql.createPool(config); + + return { + dialect: MYSQL_DIALECT, + + async exec(sql) { + // DDL only, and never parameterised, so the text goes as-is. + await pool.query(sql); + }, + + async run(sql, params) { + await pool.execute(sql, params); + }, + + async all(sql: string, params: unknown[]) { + const [rows] = await pool.execute(sql, params); + return rows as T[]; + }, + + async close() { + await pool.end(); + }, + }; +} diff --git a/src/store/drivers/optional.ts b/src/store/drivers/optional.ts new file mode 100644 index 0000000..8c1e4b7 --- /dev/null +++ b/src/store/drivers/optional.ts @@ -0,0 +1,33 @@ +import { DriverNotInstalledError, type DbScheme } from '../types.js'; + +/** + * Import a driver that may not be installed. + * + * The specifier is held in a variable rather than written as a literal so that + * TypeScript does not try to resolve the module at build time. These are + * optional peer dependencies — the package builds and runs fine with none of + * them present, and a user installs only the one backend they actually use. + * + * A failed import is reported as an actionable install instruction rather than a + * module-resolution stack trace. + */ +export async function importOptional>( + specifier: string, + scheme: DbScheme, + packageName: string = specifier, +): Promise { + let loaded: unknown; + try { + loaded = await import(/* @vite-ignore */ specifier); + } catch (error) { + throw new DriverNotInstalledError(scheme, packageName, error); + } + + // Several of these drivers are CommonJS, so the useful export sits on + // `.default` once Node's interop has wrapped it. + const namespace = loaded as { default?: unknown }; + if (namespace.default && typeof namespace.default === 'object') { + return { ...(namespace.default as object), ...(loaded as object) } as T; + } + return loaded as T; +} diff --git a/src/store/drivers/postgres.ts b/src/store/drivers/postgres.ts new file mode 100644 index 0000000..1ae8aa1 --- /dev/null +++ b/src/store/drivers/postgres.ts @@ -0,0 +1,121 @@ +import { NUMBERED_DIALECT, type SqlDriver } from '../adapters/sql.js'; +import { importOptional } from './optional.js'; +import type { ParsedDsn } from '../dsn.js'; + +type SslOption = false | { rejectUnauthorized: boolean } | undefined; + +/** + * Translate `sslmode` into what `pg` expects. + * + * Managed Postgres — Google Cloud SQL, Azure Database for PostgreSQL, Neon, + * Supabase, RDS — generally requires TLS but presents a chain Node does not + * trust out of the box, so `require` encrypts without verifying. Callers who + * want verification ask for it explicitly with `verify-ca` or `verify-full`. + */ +function sslOptionFor(mode: string | undefined): SslOption { + switch (mode) { + case 'disable': + return false; + case 'require': + case 'prefer': + case 'allow': + return { rejectUnauthorized: false }; + case 'verify-ca': + case 'verify-full': + return { rejectUnauthorized: true }; + default: + return undefined; + } +} + +type PgPool = { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }>; end: () => Promise }; + +function loadPg(scheme: 'postgres' | 'cloudsql') { + return importOptional<{ Pool: new (config: unknown) => PgPool }>('pg', scheme, 'pg'); +} + +function toDriver(pool: PgPool, name: string): SqlDriver { + return { + dialect: NUMBERED_DIALECT(name), + + async exec(sql) { + await pool.query(sql); + }, + + async run(sql, params) { + await pool.query(sql, params); + }, + + async all(sql: string, params: unknown[]) { + const result = await pool.query(sql, params); + return result.rows as T[]; + }, + + async close() { + await pool.end(); + }, + }; +} + +/** + * Postgres over the wire. + * + * This covers self-hosted Postgres and every managed flavour that exposes a + * standard endpoint, including Azure Database for PostgreSQL and Cloud SQL + * reached by IP. Cloud SQL reached by instance connection name uses + * `createCloudSqlDriver` below instead. + */ +export async function createPostgresDriver(dsn: ParsedDsn): Promise { + const pg = await loadPg('postgres'); + + const ssl = sslOptionFor(dsn.params.sslmode); + const config: Record = { connectionString: dsn.raw }; + if (ssl !== undefined) { + config.ssl = ssl; + } + + return toDriver(new pg.Pool(config), 'postgres'); +} + +/** + * Google Cloud SQL for PostgreSQL, addressed by instance connection name. + * + * The official connector handles TLS and, when no password is supplied, IAM + * database authentication — neither of which a plain `postgres://` URL can do. + * It is what makes `project:region:instance` addressing work without pinning an + * IP or running the auth proxy as a sidecar. + */ +export async function createCloudSqlDriver(dsn: ParsedDsn): Promise { + const pg = await loadPg('cloudsql'); + const { Connector } = await importOptional<{ + Connector: new () => { + getOptions(opts: Record): Promise>; + close(): void; + }; + }>('@google-cloud/cloud-sql-connector', 'cloudsql', '@google-cloud/cloud-sql-connector pg'); + + const connector = new Connector(); + const clientOpts = await connector.getOptions({ + instanceConnectionName: dsn.instance!, + ipType: (dsn.params.ipType ?? 'PUBLIC').toUpperCase(), + // No password means IAM database authentication. + authType: dsn.password ? 'PASSWORD' : 'IAM', + }); + + const pool = new pg.Pool({ + ...clientOpts, + user: dsn.username, + password: dsn.password, + database: dsn.database, + }); + + const driver = toDriver(pool, 'cloudsql'); + const closePool = driver.close.bind(driver); + return { + ...driver, + async close() { + await closePool(); + connector.close(); + }, + }; +} diff --git a/src/store/drivers/redis.ts b/src/store/drivers/redis.ts new file mode 100644 index 0000000..f9bf874 --- /dev/null +++ b/src/store/drivers/redis.ts @@ -0,0 +1,216 @@ +import type { Collection, Document, DocumentDriver } from '../adapters/document.js'; +import type { ParsedDsn } from '../dsn.js'; +import { InvalidDsnError } from '../types.js'; +import { importOptional } from './optional.js'; + +const KEYS: Record = { + contexts: 'opencontext:contexts', + bubbles: 'opencontext:bubbles', +}; + +interface RedisClient { + readonly isOpen: boolean; + readonly isReady: boolean; + on(event: string, listener: (...args: never[]) => void): unknown; + connect(): Promise; + ping(): Promise; + /** node-redis v5+. Older releases only have `quit`. */ + close?: () => Promise; + quit(): Promise; + /** node-redis v5+. Older releases call the same thing `disconnect`. */ + destroy?: () => void; + disconnect?: () => Promise; + hGet(key: string, field: string): Promise; + hSet(key: string, field: string, value: string): Promise; + hDel(key: string, field: string): Promise; + hGetAll(key: string): Promise>; +} + +/** + * Rewrite the connection string into something node-redis will accept. + * + * node-redis parses the URL itself and rejects any scheme other than `redis:` + * and `rediss:` with a bare `TypeError: Invalid protocol`. `valkey://` is an + * alias opencontext advertises, so the scheme is swapped out here rather than + * leaking a driver-internal error to someone who typed a documented URL. TLS + * stays on only for `rediss://`; everything else connects in the clear. + */ +function clientUrl(dsn: ParsedDsn): string { + // Every other network backend here names its database; Redis numbers them, + // and node-redis answers a name with a bare `TypeError: Invalid pathname`. + if (dsn.database !== undefined && !/^\d+$/.test(dsn.database)) { + throw new InvalidDsnError( + `Redis addresses a numbered database, so "${dsn.database}" is not a valid ` + + 'database (e.g. redis://HOST:6379/0).', + ); + } + const secure = /^rediss:/i.test(dsn.raw); + return dsn.raw.replace(/^[a-zA-Z][a-zA-Z0-9+.-]*:/, secure ? 'rediss:' : 'redis:'); +} + +/** + * Values are JSON strings, and a hash field can hold anything — including + * something another program wrote. Report which field is bad rather than + * letting a bare `SyntaxError` out of `listContexts`, where it would look like + * an opencontext failure and give no hint what to delete. + */ +function parseDocument(collection: Collection, id: string, raw: string): Document { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error( + `Redis hash field ${KEYS[collection]}[${id}] does not contain valid JSON. ` + + 'Delete that field, or point opencontext at a database it owns.', + ); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error( + `Redis hash field ${KEYS[collection]}[${id}] does not contain a JSON object.`, + ); + } + return parsed as Document; +} + +/** How many times to retry before deciding the store was never reachable. */ +const CONNECT_ATTEMPTS = 3; + +/** + * Redis, and anything speaking its protocol — Valkey, Upstash, ElastiCache. + * + * Each collection is one hash keyed by id, so reads and writes are O(1) and + * listing a collection is a single `HGETALL`. Values are JSON strings, which + * keeps this working on a stock Redis with no modules installed. + * + * `HGETALL` is O(N) over the collection and reads it in one shot. That matches + * what the shared document adapter needs — every search reads the whole + * collection anyway — and it buys an atomic snapshot that `HSCAN` cannot give, + * since a scan can return the same field twice while writes are in flight. The + * cost is that a very large store blocks the server for the length of the call; + * a SQL backend is the better choice at that size. + */ +export async function createRedisDriver(dsn: ParsedDsn): Promise { + // Checked before the driver is loaded, so a bad URL is reported as a bad URL + // rather than as a missing npm package. + const url = clientUrl(dsn); + + const { createClient } = await importOptional<{ + createClient: (config: Record) => RedisClient; + }>('redis', 'redis'); + + /** True once the socket has been usable at least once. */ + let everReady = false; + + const client = createClient({ + url, + socket: { + reconnectStrategy(retries: number, cause: Error): number | false { + // node-redis retries the *first* connection forever by default, so a + // typo in the host or a Redis that is not running would hang + // `createStore` rather than fail it. Nothing about a URL that has never + // worked gets better by waiting, so give up and report the cause. + if (!everReady && retries >= CONNECT_ATTEMPTS) { + return false; + } + // Once it has worked, keep reconnecting — a restart or a failover is + // exactly the case a long-lived MCP or HTTP server has to ride out. + // Exponential backoff with jitter, matching node-redis' own default. + return Math.min(2 ** retries * 50, 2000) + Math.floor(Math.random() * 200); + }, + }, + }); + + // node-redis emits `error` for every socket failure and reconnect attempt. An + // EventEmitter with no `error` listener throws, which would take down the + // whole host process — the MCP server, the HTTP server — the moment Redis + // blinks. Hold the last one instead, so `ping` can report why the store is + // unhealthy while the client reconnects underneath. + let lastSocketError: Error | undefined; + client.on('error', (error: Error) => { + lastSocketError = error; + }); + client.on('ready', () => { + everReady = true; + lastSocketError = undefined; + }); + + /** Set once the caller has closed the store, so shutdown stays idempotent. */ + let closed = false; + + async function shutdown(): Promise { + try { + // `quit` is deprecated in favour of `close` from node-redis v5 on; both + // wait for in-flight commands. + await (client.close ? client.close() : client.quit()); + } catch { + // The socket was already gone, so there was nobody to answer QUIT. Drop + // the client outright — a shutdown path must not throw, and a client left + // half-open keeps a reconnect timer and the process alive with it. + try { + client.destroy?.(); + } catch { + // Already destroyed. + } + } + } + + return { + async connect() { + if (client.isOpen) { + return; + } + try { + await client.connect(); + } catch (error) { + // A connect that never succeeded still leaves a socket and a retry + // timer behind. Tear them down before reporting, or the process hangs + // long after the caller has given up. + try { + client.destroy?.(); + } catch { + // Nothing was open in the first place. + } + closed = true; + throw error; + } + }, + + async close() { + if (closed) { + return; + } + closed = true; + await shutdown(); + }, + + async ping() { + // A command issued while the socket is down sits in the offline queue + // until the reconnect succeeds. That is right for writes and wrong for a + // health check, which is being asked whether the store is reachable now. + if (!client.isReady) { + throw lastSocketError ?? new Error('Redis connection is not ready.'); + } + await client.ping(); + }, + + async get(collection, id) { + const raw = await client.hGet(KEYS[collection], id); + return raw === null || raw === undefined + ? undefined + : parseDocument(collection, id, raw); + }, + + async put(collection, id, document) { + await client.hSet(KEYS[collection], id, JSON.stringify(document)); + }, + + async remove(collection, id) { + await client.hDel(KEYS[collection], id); + }, + + async list(collection) { + const fields = await client.hGetAll(KEYS[collection]); + return Object.entries(fields).map(([id, raw]) => parseDocument(collection, id, raw)); + }, + }; +} diff --git a/src/store/drivers/sqlite.ts b/src/store/drivers/sqlite.ts new file mode 100644 index 0000000..220f951 --- /dev/null +++ b/src/store/drivers/sqlite.ts @@ -0,0 +1,92 @@ +import { existsSync, mkdirSync } from 'fs'; +import { dirname } from 'path'; +import { QUESTION_MARK_DIALECT, type SqlDriver } from '../adapters/sql.js'; +import { importOptional } from './optional.js'; +import type { ParsedDsn } from '../dsn.js'; + +/** + * Local SQLite via `node:sqlite`, which ships with Node — no dependency to + * install, which is why SQLite is the recommended first step up from JSON. + * + * The module is synchronous; every method is wrapped in a promise so it satisfies + * the same `SqlDriver` contract as the genuinely async drivers. + */ +export async function createSqliteDriver(dsn: ParsedDsn): Promise { + const { DatabaseSync } = await import('node:sqlite'); + + const path = dsn.path!; + if (path !== ':memory:') { + const directory = dirname(path); + if (directory && !existsSync(directory)) { + mkdirSync(directory, { recursive: true }); + } + } + + const db = new DatabaseSync(path); + db.exec('PRAGMA journal_mode = WAL'); + db.exec('PRAGMA foreign_keys = ON'); + + return { + dialect: QUESTION_MARK_DIALECT('sqlite'), + + async exec(sql) { + db.exec(sql); + }, + + async run(sql, params) { + db.prepare(sql).run(...(params as never[])); + }, + + async all(sql: string, params: unknown[]) { + return db.prepare(sql).all(...(params as never[])) as T[]; + }, + + async close() { + db.close(); + }, + }; +} + +interface LibsqlClient { + execute(statement: string | { sql: string; args: unknown[] }): Promise<{ rows: unknown[] }>; + close(): void; +} + +/** + * Remote SQLite over libSQL (Turso and self-hosted sqld). + * + * Same dialect as local SQLite — only the transport differs. + */ +export async function createLibsqlDriver(dsn: ParsedDsn): Promise { + const { createClient } = await importOptional<{ + createClient: (config: Record) => LibsqlClient; + }>('@libsql/client', 'libsql'); + + const authToken = dsn.params.authToken ?? dsn.params.auth_token; + // Strip the token from the URL — libsql takes it as a separate option and + // would otherwise see it twice. + const url = dsn.raw.replace(/[?&](authToken|auth_token)=[^&]*/g, '').replace(/\?$/, ''); + + const client = createClient(authToken ? { url, authToken } : { url }); + + return { + dialect: QUESTION_MARK_DIALECT('libsql'), + + async exec(sql) { + await client.execute(sql); + }, + + async run(sql, params) { + await client.execute({ sql, args: params as never[] }); + }, + + async all(sql: string, params: unknown[]) { + const result = await client.execute({ sql, args: params as never[] }); + return result.rows as unknown as T[]; + }, + + async close() { + client.close(); + }, + }; +} diff --git a/src/store/dsn.ts b/src/store/dsn.ts new file mode 100644 index 0000000..1f7e711 --- /dev/null +++ b/src/store/dsn.ts @@ -0,0 +1,402 @@ +import { InvalidDsnError, type DbScheme } from './types.js'; + +export const SUPPORTED_SCHEMES: DbScheme[] = [ + 'json', + 'memory', + 'sqlite', + 'duckdb', + 'libsql', + 'd1', + 'postgres', + 'cloudsql', + 'mysql', + 'mssql', + 'mongodb', + 'redis', + 'firestore', + 'dynamodb', + 'surrealdb', +]; + +/** Schemes that address a file on disk rather than a network endpoint. */ +const FILE_SCHEMES = new Set(['json', 'memory', 'sqlite', 'duckdb']); + +/** Alternate spellings users reasonably expect to work. */ +const SCHEME_ALIASES: Record = { + postgresql: 'postgres', + ws: 'surrealdb', + wss: 'surrealdb', + surreal: 'surrealdb', + sqlserver: 'mssql', + azuresql: 'mssql', + ddb: 'dynamodb', + mariadb: 'mysql', + 'mongodb+srv': 'mongodb', + mongo: 'mongodb', + rediss: 'redis', + valkey: 'redis', +}; + +const DEFAULT_PORTS: Partial> = { + postgres: 5432, + mysql: 3306, + mssql: 1433, + mongodb: 27017, + redis: 6379, + surrealdb: 8000, +}; + +/** + * The scheme spelling each driver library actually accepts. + * + * This is not simply the normalised `DbScheme`: `rediss` and `mongodb+srv` carry + * meaning that must survive. Collapsing `rediss://` to `redis://` would silently + * turn TLS off, and dropping `+srv` would turn an Atlas SRV lookup into a direct + * connection to a host that does not answer. + */ +const CANONICAL_SCHEMES: Partial string>> = { + redis: (original) => (original === 'rediss' ? 'rediss' : 'redis'), + mongodb: (original) => (original === 'mongodb+srv' ? 'mongodb+srv' : 'mongodb'), + postgres: () => 'postgres', +}; + +function canonicalise(raw: string, original: string, scheme: DbScheme): string { + const resolver = CANONICAL_SCHEMES[scheme]; + if (!resolver) { + return raw; + } + const canonicalScheme = resolver(original); + return `${canonicalScheme}:${raw.slice(original.length + 1)}`; +} + +/** Query parameter names whose values are secrets. */ +const SECRET_PARAMS = ['authtoken', 'token', 'password', 'apikey', 'api_key']; + +export interface ParsedDsn { + scheme: DbScheme; + /** The original string, credentials intact. Never log this. */ + raw: string; + /** + * `raw` with the scheme rewritten to the exact spelling the driver library + * expects. Drivers must pass this, not `raw` — the client libraries reject + * aliases we advertise (`mongo://`, `valkey://`) and are case-sensitive. + */ + canonical: string; + /** Same string with credentials masked. Safe to log and to send to the UI. */ + redacted: string; + remote: boolean; + /** File-based schemes only. */ + path?: string; + host?: string; + port?: number; + username?: string; + password?: string; + database?: string; + /** SurrealDB only. */ + namespace?: string; + /** SurrealDB only — the http(s) endpoint derived from the connection string. */ + endpoint?: string; + /** Cloud SQL only — the `project:region:instance` connection name. */ + instance?: string; + /** DynamoDB only — the AWS region and table name. */ + region?: string; + table?: string; + /** Cloudflare D1 only. */ + accountId?: string; + databaseId?: string; + /** Firestore only — the GCP project. */ + project?: string; + params: Record; +} + +function splitScheme(input: string): { scheme: string; rest: string } | undefined { + const match = /^([a-zA-Z][a-zA-Z0-9+.-]*):(.*)$/.exec(input); + if (!match) { + return undefined; + } + return { scheme: match[1]!.toLowerCase(), rest: match[2]! }; +} + +function normalizeScheme(scheme: string): DbScheme { + const resolved = SCHEME_ALIASES[scheme] ?? (scheme as DbScheme); + if (!SUPPORTED_SCHEMES.includes(resolved)) { + throw new InvalidDsnError( + `Unsupported database scheme "${scheme}". ` + + `Supported schemes: ${SUPPORTED_SCHEMES.join(', ')}.`, + ); + } + return resolved; +} + +function parseFileDsn(scheme: DbScheme, rest: string, raw: string): ParsedDsn { + // `memory://` is process-local and ephemeral. Anything after the scheme names + // an independent store, so `memory://scratch` and `memory://` do not collide. + if (scheme === 'memory') { + const name = (rest.startsWith('//') ? rest.slice(2) : rest) || 'default'; + return { scheme, raw, canonical: raw, redacted: raw, remote: false, path: name, params: {} }; + } + // `sqlite::memory:` — the rest is the literal `:memory:` marker. + if (rest === ':memory:') { + return { scheme, raw, canonical: raw, redacted: raw, remote: false, path: ':memory:', params: {} }; + } + // `json:///abs/path` → `//` authority prefix, empty host, path follows. + const path = rest.startsWith('//') ? rest.slice(2) : rest; + if (!path) { + throw new InvalidDsnError( + `${scheme} connection string is missing a file path (e.g. ${scheme}:///path/to/store).`, + ); + } + return { scheme, raw, canonical: raw, redacted: raw, remote: false, path, params: {} }; +} + +function collectParams(url: URL): Record { + const params: Record = {}; + url.searchParams.forEach((value, key) => { + params[key] = value; + }); + return params; +} + +/** + * Google Cloud SQL: `cloudsql://user:password@PROJECT:REGION:INSTANCE/DATABASE`. + * + * Parsed by hand rather than with `URL`, because a Cloud SQL instance connection + * name contains colons and the WHATWG parser would read the first one as a port + * separator. + */ +function parseCloudSqlDsn(raw: string): ParsedDsn { + const match = + /^cloudsql:\/\/(?:([^:@/]+)(?::([^@/]*))?@)?([^/?#]+)\/([^/?#]+)(\?.*)?$/.exec(raw); + if (!match) { + throw new InvalidDsnError( + 'Cloud SQL connection string must look like ' + + 'cloudsql://user:password@project:region:instance/database.', + ); + } + const [, username, password, instance, database, query] = match; + + if ((instance!.match(/:/g) ?? []).length !== 2) { + throw new InvalidDsnError( + `Cloud SQL instance "${instance}" must be a full connection name ` + + 'in the form project:region:instance.', + ); + } + + const params: Record = {}; + if (query) { + new URLSearchParams(query.slice(1)).forEach((value, key) => { + params[key] = value; + }); + } + + const parsed: ParsedDsn = { + scheme: 'cloudsql', + raw, + canonical: raw, + redacted: redactDsn(raw), + remote: true, + instance: instance!, + database: decodeURIComponent(database!), + params, + }; + if (username) { + parsed.username = decodeURIComponent(username); + } + if (password) { + parsed.password = decodeURIComponent(password); + } + return parsed; +} + +function parseNetworkDsn(scheme: DbScheme, raw: string, original: string): ParsedDsn { + let url: URL; + try { + // Swap in a neutral scheme so the WHATWG parser applies generic rules + // consistently rather than protocol-specific ones. + url = new URL(raw.replace(/^[a-zA-Z][a-zA-Z0-9+.-]*:/, 'http:')); + } catch { + throw new InvalidDsnError(`Could not parse ${scheme} connection string.`); + } + + const segments = url.pathname.split('/').filter(Boolean); + const parsed: ParsedDsn = { + scheme, + raw, + canonical: canonicalise(raw, original, scheme), + redacted: redactDsn(raw), + remote: true, + host: url.hostname, + port: url.port ? parseInt(url.port, 10) : DEFAULT_PORTS[scheme], + params: collectParams(url), + }; + + if (url.username) { + parsed.username = decodeURIComponent(url.username); + } + if (url.password) { + parsed.password = decodeURIComponent(url.password); + } + + if (scheme === 'surrealdb') { + if (segments.length < 2) { + throw new InvalidDsnError( + 'SurrealDB connection string needs both a namespace and a database ' + + '(e.g. surrealdb://user:pass@host:8000/namespace/database).', + ); + } + parsed.namespace = decodeURIComponent(segments[0]!); + parsed.database = decodeURIComponent(segments[1]!); + const secure = raw.startsWith('wss:') || raw.startsWith('https:'); + const authority = url.port ? `${url.hostname}:${url.port}` : url.hostname; + parsed.endpoint = `${secure ? 'https' : 'http'}://${authority}`; + return parsed; + } + + if (scheme === 'd1') { + // `d1://ACCOUNT_ID/DATABASE_ID?apiToken=…` + if (!url.hostname || segments.length < 1) { + throw new InvalidDsnError( + 'Cloudflare D1 connection string needs an account id and a database id ' + + '(e.g. d1://ACCOUNT_ID/DATABASE_ID?apiToken=…).', + ); + } + parsed.accountId = url.hostname; + parsed.databaseId = decodeURIComponent(segments[0]!); + delete parsed.port; + return parsed; + } + + if (scheme === 'firestore') { + if (!url.hostname) { + throw new InvalidDsnError( + 'Firestore connection string needs a project id (e.g. firestore://my-project).', + ); + } + parsed.project = url.hostname; + parsed.database = segments[0] ? decodeURIComponent(segments[0]) : '(default)'; + delete parsed.port; + return parsed; + } + + if (scheme === 'mongodb') { + // A database segment is optional; Mongo falls back to a default below. + parsed.database = segments[0] ? decodeURIComponent(segments[0]) : 'opencontext'; + return parsed; + } + + if (scheme === 'redis') { + // Redis addresses a numbered database, not a named one. + parsed.database = segments[0] ? decodeURIComponent(segments[0]) : '0'; + return parsed; + } + + if (scheme === 'dynamodb') { + // The host slot carries the AWS region; the first path segment the table. + if (!url.hostname || segments.length < 1) { + throw new InvalidDsnError( + 'DynamoDB connection string needs a region and a table name ' + + '(e.g. dynamodb://us-east-1/opencontext).', + ); + } + parsed.region = url.hostname; + parsed.table = decodeURIComponent(segments[0]!); + delete parsed.port; + return parsed; + } + + if (scheme === 'postgres' || scheme === 'mssql' || scheme === 'mysql') { + if (segments.length < 1) { + const examples: Record = { + postgres: 'postgres://user:pass@host:5432/opencontext', + mysql: 'mysql://user:pass@host:3306/opencontext', + mssql: 'mssql://user:pass@server.database.windows.net:1433/opencontext', + }; + const example = examples[scheme]!; + throw new InvalidDsnError( + `${scheme} connection string needs a database name (e.g. ${example}).`, + ); + } + parsed.database = decodeURIComponent(segments[0]!); + return parsed; + } + + // libsql addresses a whole database by host; a path segment is optional. + if (segments.length > 0) { + parsed.database = decodeURIComponent(segments[0]!); + } + return parsed; +} + +/** + * Parse a connection string into its parts. + * + * A string with no recognised scheme is treated as a JSON file path, which keeps + * the legacy `OPENCONTEXT_STORE_PATH` value working unchanged. + */ +export function parseDsn(input: string): ParsedDsn { + const trimmed = input.trim(); + if (!trimmed) { + throw new InvalidDsnError('Connection string is empty.'); + } + + const split = splitScheme(trimmed); + + // No scheme, or a bare Windows drive letter — treat it as a file path. + if (!split || split.scheme.length === 1) { + return { + scheme: 'json', raw: trimmed, canonical: trimmed, redacted: trimmed, + remote: false, path: trimmed, params: {}, + }; + } + + const scheme = normalizeScheme(split.scheme); + if (scheme === 'cloudsql') { + return parseCloudSqlDsn(trimmed); + } + return FILE_SCHEMES.has(scheme) + ? parseFileDsn(scheme, split.rest, trimmed) + : parseNetworkDsn(scheme, trimmed, split.scheme); +} + +/** + * Mask every credential in a connection string. Applied to anything that reaches + * a log line, an API response, or the UI. + * + * Input that cannot be parsed is returned unchanged — this is a display helper + * and must never be the thing that throws. + */ +export function redactDsn(input: string): string { + const trimmed = input.trim(); + const split = splitScheme(trimmed); + if (!split || split.rest.startsWith(':') || !split.rest.startsWith('//')) { + return input; + } + + let redacted = trimmed; + + // user:password@host → user:***@host + redacted = redacted.replace( + /^([a-zA-Z][a-zA-Z0-9+.-]*:\/\/[^/?#@]*?:)[^/?#@]*(@)/, + '$1***$2', + ); + + // ?authToken=secret → ?authToken=*** + const queryStart = redacted.indexOf('?'); + if (queryStart !== -1) { + const query = redacted + .slice(queryStart + 1) + .split('&') + .map((pair) => { + const eq = pair.indexOf('='); + if (eq === -1) { + return pair; + } + const key = pair.slice(0, eq); + return SECRET_PARAMS.includes(key.toLowerCase()) ? `${key}=***` : pair; + }) + .join('&'); + redacted = `${redacted.slice(0, queryStart)}?${query}`; + } + + return redacted; +} diff --git a/src/store/index.ts b/src/store/index.ts new file mode 100644 index 0000000..aa0819b --- /dev/null +++ b/src/store/index.ts @@ -0,0 +1,180 @@ +import { + InvalidDsnError, + type ContextStoreAdapter, + type AdapterInfo, + type DbScheme, +} from './types.js'; +import { parseDsn, redactDsn, type ParsedDsn } from './dsn.js'; +import { createSqlAdapter } from './adapters/sql.js'; +import { createDocumentAdapter } from './adapters/document.js'; +import { createJsonAdapter } from './adapters/json.js'; +import { createSurrealAdapter } from './adapters/surreal.js'; + +export interface AdapterDescriptor { + scheme: DbScheme; + label: string; + /** What the user types, with the parts they must replace spelled out. */ + example: string; + /** npm package needed to use it, or null when nothing needs installing. */ + packageName: string | null; + remote: boolean; + family: 'file' | 'sql' | 'document'; +} + +/** + * Every backend opencontext can open. + * + * `packageName: null` means the backend works out of the box — either it uses + * only Node built-ins (json, memory, sqlite) or it speaks plain HTTP (d1). + */ +export const ADAPTERS: AdapterDescriptor[] = [ + { scheme: 'json', label: 'JSON file', example: 'json:///path/to/contexts.json', packageName: null, remote: false, family: 'file' }, + { scheme: 'memory', label: 'In-memory (ephemeral)', example: 'memory://', packageName: null, remote: false, family: 'document' }, + { scheme: 'sqlite', label: 'SQLite', example: 'sqlite:///path/to/opencontext.db', packageName: null, remote: false, family: 'sql' }, + { scheme: 'duckdb', label: 'DuckDB', example: 'duckdb:///path/to/opencontext.duckdb', packageName: '@duckdb/node-api', remote: false, family: 'sql' }, + { scheme: 'libsql', label: 'libSQL / Turso', example: 'libsql://DATABASE.turso.io?authToken=TOKEN', packageName: '@libsql/client', remote: true, family: 'sql' }, + { scheme: 'd1', label: 'Cloudflare D1', example: 'd1://ACCOUNT_ID/DATABASE_ID?apiToken=TOKEN', packageName: null, remote: true, family: 'sql' }, + { scheme: 'postgres', label: 'PostgreSQL', example: 'postgres://USER:PASSWORD@HOST:5432/DATABASE', packageName: 'pg', remote: true, family: 'sql' }, + { scheme: 'cloudsql', label: 'Google Cloud SQL', example: 'cloudsql://USER:PASSWORD@PROJECT:REGION:INSTANCE/DATABASE', packageName: '@google-cloud/cloud-sql-connector', remote: true, family: 'sql' }, + { scheme: 'mysql', label: 'MySQL / MariaDB', example: 'mysql://USER:PASSWORD@HOST:3306/DATABASE', packageName: 'mysql2', remote: true, family: 'sql' }, + { scheme: 'mssql', label: 'SQL Server / Azure SQL', example: 'mssql://USER:PASSWORD@HOST:1433/DATABASE', packageName: 'mssql', remote: true, family: 'sql' }, + { scheme: 'mongodb', label: 'MongoDB', example: 'mongodb://USER:PASSWORD@HOST:27017/DATABASE', packageName: 'mongodb', remote: true, family: 'document' }, + { scheme: 'redis', label: 'Redis / Valkey', example: 'redis://HOST:6379', packageName: 'redis', remote: true, family: 'document' }, + { scheme: 'firestore', label: 'Google Firestore', example: 'firestore://PROJECT_ID', packageName: '@google-cloud/firestore', remote: true, family: 'document' }, + { scheme: 'dynamodb', label: 'Amazon DynamoDB', example: 'dynamodb://REGION/TABLE', packageName: '@aws-sdk/client-dynamodb', remote: true, family: 'document' }, + { scheme: 'surrealdb', label: 'SurrealDB', example: 'surrealdb://USER:PASSWORD@HOST:8000/NAMESPACE/DATABASE', packageName: 'surrealdb', remote: true, family: 'document' }, +]; + +export function describeAdapter(scheme: DbScheme): AdapterDescriptor { + const found = ADAPTERS.find((adapter) => adapter.scheme === scheme); + if (!found) { + throw new InvalidDsnError(`No adapter registered for scheme "${scheme}".`); + } + return found; +} + +/** Is the optional driver for this backend importable right now? */ +export async function isDriverInstalled(scheme: DbScheme): Promise { + const { packageName } = describeAdapter(scheme); + if (!packageName) { + return true; + } + // Only the first package is probed; the rest install alongside it. + const specifier = packageName.split(' ')[0]!; + try { + await import(/* @vite-ignore */ specifier); + return true; + } catch { + return false; + } +} + +function infoFor(dsn: ParsedDsn): AdapterInfo { + const descriptor = describeAdapter(dsn.scheme); + return { + scheme: dsn.scheme, + label: descriptor.label, + target: dsn.path ?? redactDsn(dsn.raw), + remote: dsn.remote, + }; +} + +/** + * Build an adapter for a connection string. + * + * Each driver is imported only when its scheme is actually used, so nothing pays + * for backends it does not touch — including at build time, where none of the + * optional packages need to be present. + */ +async function build(dsn: ParsedDsn): Promise { + const info = infoFor(dsn); + + switch (dsn.scheme) { + // ---- file ------------------------------------------------------------ + case 'json': + return createJsonAdapter(dsn); + + // ---- SQL ------------------------------------------------------------- + case 'sqlite': { + const { createSqliteDriver } = await import('./drivers/sqlite.js'); + return createSqlAdapter(await createSqliteDriver(dsn), info); + } + case 'libsql': { + const { createLibsqlDriver } = await import('./drivers/sqlite.js'); + return createSqlAdapter(await createLibsqlDriver(dsn), info); + } + case 'd1': { + const { createD1Driver } = await import('./drivers/d1.js'); + return createSqlAdapter(await createD1Driver(dsn), info); + } + case 'duckdb': { + const { createDuckDbDriver } = await import('./drivers/duckdb.js'); + return createSqlAdapter(await createDuckDbDriver(dsn), info); + } + case 'postgres': { + const { createPostgresDriver } = await import('./drivers/postgres.js'); + return createSqlAdapter(await createPostgresDriver(dsn), info); + } + case 'cloudsql': { + const { createCloudSqlDriver } = await import('./drivers/postgres.js'); + return createSqlAdapter(await createCloudSqlDriver(dsn), info); + } + case 'mysql': { + const { createMysqlDriver } = await import('./drivers/mysql.js'); + return createSqlAdapter(await createMysqlDriver(dsn), info); + } + case 'mssql': { + const { createMssqlDriver } = await import('./drivers/mssql.js'); + return createSqlAdapter(await createMssqlDriver(dsn), info); + } + + // ---- document / key-value ------------------------------------------- + case 'memory': { + const { createMemoryDriver } = await import('./drivers/memory.js'); + // `memory://scratch` names an independent store; `memory://` is the default. + const name = dsn.path?.replace(/^\/*/, '') || 'default'; + return createDocumentAdapter(createMemoryDriver(name), info); + } + case 'mongodb': { + const { createMongoDriver } = await import('./drivers/mongodb.js'); + return createDocumentAdapter(await createMongoDriver(dsn), info); + } + case 'redis': { + const { createRedisDriver } = await import('./drivers/redis.js'); + return createDocumentAdapter(await createRedisDriver(dsn), info); + } + case 'firestore': { + const { createFirestoreDriver } = await import('./drivers/firestore.js'); + return createDocumentAdapter(await createFirestoreDriver(dsn), info); + } + case 'dynamodb': { + const { createDynamoDbDriver } = await import('./drivers/dynamodb.js'); + return createDocumentAdapter(await createDynamoDbDriver(dsn), info); + } + + // ---- multi-model ----------------------------------------------------- + case 'surrealdb': + return createSurrealAdapter(dsn, info); + + default: { + const exhaustive: never = dsn.scheme; + throw new InvalidDsnError(`Unsupported scheme "${String(exhaustive)}".`); + } + } +} + +/** + * Open a connected store for a connection string. + * + * The adapter is returned already connected, so callers never have to remember + * to call `connect()` and no backend can be used half-initialised. + */ +export async function createStore(url: string): Promise { + const dsn = parseDsn(url); + const adapter = await build(dsn); + await adapter.connect(); + return adapter; +} + +export { parseDsn, redactDsn } from './dsn.js'; +export * from './types.js'; diff --git a/src/store/manager.ts b/src/store/manager.ts new file mode 100644 index 0000000..b65999f --- /dev/null +++ b/src/store/manager.ts @@ -0,0 +1,80 @@ +import type { ContextStoreAdapter, AdapterInfo } from './types.js'; +import { createStore } from './index.js'; +import { resolveDatabase, writeDatabaseUrl, type ResolvedDatabase } from './config.js'; + +export interface StoreManager { + /** The live adapter, connecting on first use. */ + get(): Promise; + /** Where the current connection string came from. */ + resolution(): ResolvedDatabase; + /** Swap to a different backend, keeping the old one if the new one fails. */ + reconnect(url: string, options?: { persist?: boolean }): Promise; + close(): Promise; +} + +/** + * Owns the live store connection. + * + * Connection is lazy rather than eager so that importing `server.ts` stays + * synchronous — the test suite imports the Express app directly, and a top-level + * await there would change module semantics for every existing test. + */ +export function createStoreManager(): StoreManager { + let resolved = resolveDatabase(); + let adapter: ContextStoreAdapter | undefined; + let opening: Promise | undefined; + + async function open(url: string): Promise { + return createStore(url); + } + + return { + async get() { + if (adapter) { + return adapter; + } + // Collapse concurrent first-use into one connect rather than racing. + if (!opening) { + opening = open(resolved.url) + .then((opened) => { + adapter = opened; + return opened; + }) + .finally(() => { + opening = undefined; + }); + } + return opening; + }, + + resolution() { + return resolved; + }, + + async reconnect(url, options = {}) { + // Open the replacement *before* touching the current one, so a bad + // connection string typed into the settings page cannot take the store + // down — it fails, the old connection keeps serving, and the error is + // returned to the caller. + const next = await open(url); + + const previous = adapter; + adapter = next; + resolved = { url, redacted: next.info.target, source: 'config-file', locked: false }; + + if (options.persist) { + writeDatabaseUrl(url); + } + if (previous) { + await previous.close().catch(() => undefined); + } + return next.info; + }, + + async close() { + const current = adapter; + adapter = undefined; + await current?.close(); + }, + }; +} diff --git a/src/store/migrate.ts b/src/store/migrate.ts new file mode 100644 index 0000000..8cf105f --- /dev/null +++ b/src/store/migrate.ts @@ -0,0 +1,68 @@ +import type { ContextStoreAdapter } from './types.js'; + +export interface MigrationResult { + contexts: number; + bubbles: number; +} + +export interface MigrateOptions { + /** + * `copy` adds to whatever the target already holds (the default). + * `replace` empties the target first. + */ + mode?: 'copy' | 'replace'; +} + +async function clear(target: ContextStoreAdapter): Promise { + for (const bubble of await target.listBubbles()) { + await target.deleteBubble(bubble.id, true); + } + for (const entry of await target.listContexts()) { + await target.deleteContext(entry.id); + } +} + +/** + * Copy every context and bubble from one store into another. + * + * The source is only ever read, so a migration cannot damage the data the user + * already has — if the target write fails halfway, the original store is still + * intact and the operation can simply be retried. + * + * Everything is read from the source *before* the target is touched. That + * ordering is what makes `replace` safe when both handles happen to point at the + * same database: clearing first would delete the very rows about to be copied. + * + * Bubbles are written first so that contexts referencing them land on a target + * where those bubbles already exist. + */ +export async function migrateStore( + source: ContextStoreAdapter, + target: ContextStoreAdapter, + options: MigrateOptions = {}, +): Promise { + const { mode = 'copy' } = options; + + const bubbles = await source.listBubbles(); + const contexts = await source.listContexts(); + + if (mode === 'replace') { + await clear(target); + } + + // Ids are regenerated by the target's own `create` calls, so a map carries the + // source id forward to the new one and keeps context→bubble links correct. + const bubbleIdMap = new Map(); + + for (const bubble of bubbles) { + const created = await target.createBubble(bubble.name, bubble.description); + bubbleIdMap.set(bubble.id, created.id); + } + + for (const entry of contexts) { + const mappedBubbleId = entry.bubbleId ? bubbleIdMap.get(entry.bubbleId) : undefined; + await target.saveContext(entry.content, entry.tags, entry.source, mappedBubbleId); + } + + return { contexts: contexts.length, bubbles: bubbles.length }; +} diff --git a/src/store/types.ts b/src/store/types.ts new file mode 100644 index 0000000..f119b2c --- /dev/null +++ b/src/store/types.ts @@ -0,0 +1,95 @@ +import type { ContextEntry, Bubble } from '../mcp/types.js'; + +export type { ContextEntry, Bubble }; + +/** Every connection-string scheme opencontext knows how to open. */ +export type DbScheme = + // file / embedded + | 'json' + | 'memory' + | 'sqlite' + | 'duckdb' + // SQL over the wire + | 'libsql' + | 'd1' + | 'postgres' + | 'cloudsql' + | 'mysql' + | 'mssql' + // document / key-value + | 'mongodb' + | 'redis' + | 'firestore' + | 'dynamodb' + // multi-model + | 'surrealdb'; + +/** Describes the live backend. `target` is always redacted — it is sent to the UI. */ +export interface AdapterInfo { + scheme: DbScheme; + label: string; + target: string; + remote: boolean; +} + +/** + * The storage contract. Method signatures mirror the original synchronous JSON + * store; only the return types changed, so call sites just gained an `await`. + * + * Ordering contract: every list-returning method orders by `createdAt` ascending, + * then `id` ascending. This is identical across all adapters. + */ +export interface ContextStoreAdapter { + readonly info: AdapterInfo; + + connect(): Promise; + close(): Promise; + ping(): Promise; + + saveContext( + content: string, + tags?: string[], + source?: string, + bubbleId?: string, + ): Promise; + recallContext(query: string): Promise; + listContexts(tag?: string): Promise; + listContextsByBubble(bubbleId: string): Promise; + getContext(id: string): Promise; + updateContext( + id: string, + content: string, + tags?: string[], + bubbleId?: string | null, + ): Promise; + deleteContext(id: string): Promise; + searchContexts(query: string): Promise; + + createBubble(name: string, description?: string): Promise; + listBubbles(): Promise; + getBubble(id: string): Promise; + updateBubble(id: string, name: string, description?: string): Promise; + deleteBubble(id: string, deleteContexts?: boolean): Promise; +} + +/** Raised when a DSN names an adapter whose optional driver is not installed. */ +export class DriverNotInstalledError extends Error { + constructor( + public readonly scheme: DbScheme, + public readonly packageName: string, + public readonly reason?: unknown, + ) { + super( + `${scheme} driver is not installed.\nInstall it with: npm install ${packageName}`, + ); + this.name = 'DriverNotInstalledError'; + } +} + +/** Raised for malformed or unsupported connection strings. */ +export class InvalidDsnError extends Error { + constructor(message: string) { + super(message); + this.name = 'InvalidDsnError'; + } +} diff --git a/tests/mcp/store.test.ts b/tests/mcp/store.test.ts deleted file mode 100644 index 7249d4a..0000000 --- a/tests/mcp/store.test.ts +++ /dev/null @@ -1,307 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { existsSync, rmSync, mkdirSync } from 'fs'; -import { join } from 'path'; -import { tmpdir } from 'os'; -import { randomUUID } from 'crypto'; -import { createStore } from '../../src/mcp/store.js'; - -function createTempStorePath(): string { - const dir = join(tmpdir(), `opencontext-test-${randomUUID()}`); - mkdirSync(dir, { recursive: true }); - return join(dir, 'contexts.json'); -} - -describe('Context Store', () => { - let storePath: string; - - beforeEach(() => { - storePath = createTempStorePath(); - }); - - afterEach(() => { - const dir = storePath.substring(0, storePath.lastIndexOf('/')); - if (existsSync(dir)) { - rmSync(dir, { recursive: true }); - } - }); - - describe('saveContext', () => { - it('should save a context entry and return it with an ID', () => { - const store = createStore(storePath); - const entry = store.saveContext('My favorite color is blue'); - - expect(entry.id).toBeDefined(); - expect(entry.content).toBe('My favorite color is blue'); - expect(entry.tags).toEqual([]); - expect(entry.source).toBe('chat'); - expect(entry.createdAt).toBeDefined(); - }); - - it('should save with tags and source', () => { - const store = createStore(storePath); - const entry = store.saveContext( - 'Use TypeScript for all projects', - ['preference', 'code'], - 'code-review', - ); - - expect(entry.tags).toEqual(['preference', 'code']); - expect(entry.source).toBe('code-review'); - }); - - it('should persist entries to disk', () => { - const store1 = createStore(storePath); - store1.saveContext('Entry one'); - store1.saveContext('Entry two'); - - const store2 = createStore(storePath); - const all = store2.listContexts(); - expect(all).toHaveLength(2); - }); - - it('should create the store file if it does not exist', () => { - expect(existsSync(storePath)).toBe(false); - const store = createStore(storePath); - store.saveContext('Test'); - expect(existsSync(storePath)).toBe(true); - }); - }); - - describe('recallContext', () => { - it('should find contexts matching content', () => { - const store = createStore(storePath); - store.saveContext('I prefer dark mode'); - store.saveContext('My cat is named Luna'); - store.saveContext('Dark themes are better for my eyes'); - - const results = store.recallContext('dark'); - expect(results).toHaveLength(2); - expect(results[0].content).toBe('I prefer dark mode'); - expect(results[1].content).toBe('Dark themes are better for my eyes'); - }); - - it('should find contexts matching tags', () => { - const store = createStore(storePath); - store.saveContext('Use Prettier for formatting', ['tooling']); - store.saveContext('TypeScript is preferred', ['language']); - - const results = store.recallContext('tooling'); - expect(results).toHaveLength(1); - expect(results[0].content).toBe('Use Prettier for formatting'); - }); - - it('should return empty array when no match', () => { - const store = createStore(storePath); - store.saveContext('Something unrelated'); - - const results = store.recallContext('xyz-not-found'); - expect(results).toHaveLength(0); - }); - - it('should be case-insensitive', () => { - const store = createStore(storePath); - store.saveContext('TypeScript is great'); - - const results = store.recallContext('typescript'); - expect(results).toHaveLength(1); - }); - }); - - describe('listContexts', () => { - it('should list all contexts when no tag specified', () => { - const store = createStore(storePath); - store.saveContext('One'); - store.saveContext('Two'); - store.saveContext('Three'); - - const all = store.listContexts(); - expect(all).toHaveLength(3); - }); - - it('should filter by tag', () => { - const store = createStore(storePath); - store.saveContext('A', ['work']); - store.saveContext('B', ['personal']); - store.saveContext('C', ['work', 'important']); - - const workItems = store.listContexts('work'); - expect(workItems).toHaveLength(2); - }); - - it('should return empty array for empty store', () => { - const store = createStore(storePath); - const all = store.listContexts(); - expect(all).toHaveLength(0); - }); - - it('should be case-insensitive for tag filter', () => { - const store = createStore(storePath); - store.saveContext('A', ['Work']); - - const results = store.listContexts('work'); - expect(results).toHaveLength(1); - }); - }); - - describe('deleteContext', () => { - it('should delete an existing context', () => { - const store = createStore(storePath); - const entry = store.saveContext('To be deleted'); - - const deleted = store.deleteContext(entry.id); - expect(deleted).toBe(true); - - const all = store.listContexts(); - expect(all).toHaveLength(0); - }); - - it('should return false for non-existent ID', () => { - const store = createStore(storePath); - const deleted = store.deleteContext('non-existent-id'); - expect(deleted).toBe(false); - }); - - it('should only delete the targeted context', () => { - const store = createStore(storePath); - const entry1 = store.saveContext('Keep me'); - const entry2 = store.saveContext('Delete me'); - - store.deleteContext(entry2.id); - const all = store.listContexts(); - expect(all).toHaveLength(1); - expect(all[0].id).toBe(entry1.id); - }); - }); - - describe('searchContexts', () => { - it('should find contexts matching all search terms', () => { - const store = createStore(storePath); - store.saveContext('TypeScript React project'); - store.saveContext('TypeScript Node.js backend'); - store.saveContext('Python Flask API'); - - const results = store.searchContexts('TypeScript project'); - expect(results).toHaveLength(1); - expect(results[0].content).toBe('TypeScript React project'); - }); - - it('should search across content, tags, and source', () => { - const store = createStore(storePath); - store.saveContext('Some content', ['react'], 'meeting'); - - const results = store.searchContexts('react meeting'); - expect(results).toHaveLength(1); - }); - - it('should return empty when not all terms match', () => { - const store = createStore(storePath); - store.saveContext('TypeScript is great'); - - const results = store.searchContexts('TypeScript Python'); - expect(results).toHaveLength(0); - }); - }); - - describe('getContext', () => { - it('should get a specific context by ID', () => { - const store = createStore(storePath); - const entry = store.saveContext('Find me'); - - const found = store.getContext(entry.id); - expect(found).toBeDefined(); - expect(found!.content).toBe('Find me'); - }); - - it('should return undefined for non-existent ID', () => { - const store = createStore(storePath); - const found = store.getContext('does-not-exist'); - expect(found).toBeUndefined(); - }); - }); - - describe('updateContext', () => { - it('should update content of an existing context', async () => { - const store = createStore(storePath); - const entry = store.saveContext('Original content'); - - await new Promise((resolve) => setTimeout(resolve, 2)); - const updated = store.updateContext(entry.id, 'Updated content'); - expect(updated).toBeDefined(); - expect(updated!.content).toBe('Updated content'); - expect(new Date(updated!.updatedAt) >= new Date(entry.createdAt)).toBe(true); - }); - - it('should update tags when provided', () => { - const store = createStore(storePath); - const entry = store.saveContext('Content', ['old-tag']); - - const updated = store.updateContext(entry.id, 'Content', ['new-tag']); - expect(updated!.tags).toEqual(['new-tag']); - }); - - it('should keep existing tags when tags not provided', () => { - const store = createStore(storePath); - const entry = store.saveContext('Content', ['keep-me']); - - const updated = store.updateContext(entry.id, 'New content'); - expect(updated!.tags).toEqual(['keep-me']); - }); - - it('should return undefined for non-existent ID', () => { - const store = createStore(storePath); - const result = store.updateContext('fake-id', 'content'); - expect(result).toBeUndefined(); - }); - - it('should persist updates to disk', () => { - const store1 = createStore(storePath); - const entry = store1.saveContext('Original'); - store1.updateContext(entry.id, 'Updated'); - - const store2 = createStore(storePath); - const found = store2.getContext(entry.id); - expect(found!.content).toBe('Updated'); - }); - }); - - describe('default store path', () => { - it('uses USERPROFILE when HOME is unset', () => { - const originalHome = process.env.HOME; - const originalUserProfile = process.env.USERPROFILE; - delete process.env.HOME; - process.env.USERPROFILE = tmpdir(); - - const store = createStore(); - expect(store.filePath).toContain('.opencontext'); - - process.env.HOME = originalHome; - process.env.USERPROFILE = originalUserProfile; - }); - - it('falls back to cwd when HOME and USERPROFILE are both unset', () => { - const originalHome = process.env.HOME; - const originalUserProfile = process.env.USERPROFILE; - delete process.env.HOME; - delete process.env.USERPROFILE; - - const store = createStore(); - expect(store.filePath).toContain('.opencontext'); - - process.env.HOME = originalHome; - if (originalUserProfile !== undefined) { - process.env.USERPROFILE = originalUserProfile; - } - }); - }); - - describe('save creates missing directories', () => { - it('creates parent directory when it does not exist', () => { - const base = join(tmpdir(), `opencontext-nested-${randomUUID()}`); - const nestedPath = join(base, 'sub', 'contexts.json'); - const store = createStore(nestedPath); - store.saveContext('test entry'); - expect(existsSync(nestedPath)).toBe(true); - rmSync(base, { recursive: true, force: true }); - }); - }); -}); diff --git a/tests/server.test.ts b/tests/server.test.ts index 477622b..5b716e6 100644 --- a/tests/server.test.ts +++ b/tests/server.test.ts @@ -9,6 +9,10 @@ import request from 'supertest'; // --------------------------------------------------------------------------- const mockStore = vi.hoisted(() => ({ + info: { scheme: 'json', label: 'JSON file', target: '/tmp/test/contexts.json', remote: false }, + connect: vi.fn(async () => {}), + close: vi.fn(async () => {}), + ping: vi.fn(async () => {}), listContexts: vi.fn(), saveContext: vi.fn(), searchContexts: vi.fn(), @@ -23,8 +27,22 @@ const mockStore = vi.hoisted(() => ({ deleteBubble: vi.fn(), })); -vi.mock('../src/mcp/store.js', () => ({ - createStore: vi.fn(function () { return mockStore; }), +// The store is now an async, pluggable adapter reached through a manager, so the +// mock resolves a promise and stands in for whichever backend is configured. +vi.mock('../src/store/manager.js', () => ({ + createStoreManager: vi.fn(function () { + return { + get: vi.fn(async () => mockStore), + resolution: vi.fn(() => ({ + url: 'json:///tmp/test/contexts.json', + redacted: 'json:///tmp/test/contexts.json', + source: 'default', + locked: false, + })), + reconnect: vi.fn(async () => mockStore.info), + close: vi.fn(async () => {}), + }; + }), })); const mockOllamaInstance = vi.hoisted(() => ({ list: vi.fn() })); diff --git a/tests/store/backends.test.ts b/tests/store/backends.test.ts new file mode 100644 index 0000000..f1b1f97 --- /dev/null +++ b/tests/store/backends.test.ts @@ -0,0 +1,116 @@ +import { describe, it } from 'vitest'; +import { rmSync, mkdirSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { randomUUID } from 'crypto'; +import { createStore } from '../../src/store/index.js'; +import type { ContextStoreAdapter } from '../../src/store/types.js'; +import { runStoreConformance } from './conformance.js'; + +/** + * Conformance runs for every backend that needs something running. + * + * Each block is skipped unless its connection string is in the environment, so + * `npm test` stays green on a machine with no databases while + * `docker compose -f docker-compose.test.yml up -d` plus `npm run test:backends` + * exercises all of them for real. + * + * See docker-compose.test.yml for connection strings that work out of the box. + */ + +/** Remove every row through the public API — the one wipe that works everywhere. */ +async function wipe(adapter: ContextStoreAdapter): Promise { + for (const bubble of await adapter.listBubbles()) { + await adapter.deleteBubble(bubble.id, true); + } + for (const entry of await adapter.listContexts()) { + await adapter.deleteContext(entry.id); + } +} + +/** + * Register a conformance run against a live service. + * + * The store is opened once per test and wiped first, because these backends are + * shared and persistent — unlike the temp-file adapters, which get a fresh path. + */ +function describeBackend(name: string, url: string | undefined): void { + if (!url) { + describe.skip(`${name} — store conformance (set the connection string to run)`, () => { + it('skipped', () => {}); + }); + return; + } + + runStoreConformance(name, { + async setup() { + const adapter = await createStore(url); + await wipe(adapter); + await adapter.close(); + }, + async create() { + return createStore(url); + }, + async teardown() { + const adapter = await createStore(url); + await wipe(adapter); + await adapter.close(); + }, + }); +} + +// --------------------------------------------------------------------------- +// Embedded backends that need a driver installed but no running service +// --------------------------------------------------------------------------- + +const duckdbDir = join(tmpdir(), `opencontext-duckdb-${randomUUID()}`); + +if (process.env.OPENCONTEXT_TEST_DUCKDB === '1') { + runStoreConformance('duckdb', { + async setup() { + mkdirSync(duckdbDir, { recursive: true }); + }, + async create() { + return createStore(`duckdb://${join(duckdbDir, 'oc.duckdb')}`); + }, + async teardown() { + rmSync(duckdbDir, { recursive: true, force: true }); + }, + }); +} else { + describe.skip('duckdb — store conformance (set OPENCONTEXT_TEST_DUCKDB=1 to run)', () => { + it('skipped', () => {}); + }); +} + +// --------------------------------------------------------------------------- +// In-memory — no service, no driver; proves the shared document adapter +// --------------------------------------------------------------------------- + +let memoryStore: string; + +runStoreConformance('memory', { + async setup() { + memoryStore = `test-${randomUUID()}`; + }, + async create() { + return createStore(`memory://${memoryStore}`); + }, + async teardown() { + const { resetMemoryStore } = await import('../../src/store/drivers/memory.js'); + resetMemoryStore(memoryStore); + }, +}); + +// --------------------------------------------------------------------------- +// Backends that need a running service +// --------------------------------------------------------------------------- + +describeBackend('postgres', process.env.OPENCONTEXT_TEST_POSTGRES_URL); +describeBackend('mysql', process.env.OPENCONTEXT_TEST_MYSQL_URL); +describeBackend('mssql', process.env.OPENCONTEXT_TEST_MSSQL_URL); +describeBackend('mongodb', process.env.OPENCONTEXT_TEST_MONGODB_URL); +describeBackend('redis', process.env.OPENCONTEXT_TEST_REDIS_URL); +describeBackend('surrealdb', process.env.OPENCONTEXT_TEST_SURREALDB_URL); +describeBackend('dynamodb', process.env.OPENCONTEXT_TEST_DYNAMODB_URL); +describeBackend('libsql', process.env.OPENCONTEXT_TEST_LIBSQL_URL); diff --git a/tests/store/config.test.ts b/tests/store/config.test.ts new file mode 100644 index 0000000..e49c794 --- /dev/null +++ b/tests/store/config.test.ts @@ -0,0 +1,116 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { rmSync, mkdirSync, statSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { randomUUID } from 'crypto'; +import { + resolveDatabase, + writeDatabaseUrl, + readConfig, + clearDatabaseUrl, + getConfigPath, +} from '../../src/store/config.js'; + +describe('database configuration', () => { + let dir: string; + const originalEnv = { ...process.env }; + + beforeEach(() => { + dir = join(tmpdir(), `opencontext-config-${randomUUID()}`); + mkdirSync(dir, { recursive: true }); + process.env.OPENCONTEXT_CONFIG_PATH = join(dir, 'config.json'); + delete process.env.OPENCONTEXT_DB_URL; + delete process.env.OPENCONTEXT_STORE_PATH; + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + process.env = { ...originalEnv }; + }); + + describe('resolveDatabase precedence', () => { + it('defaults to the JSON store in the home directory', () => { + const resolved = resolveDatabase(); + expect(resolved.source).toBe('default'); + expect(resolved.url.startsWith('json://')).toBe(true); + expect(resolved.url).toContain('contexts.json'); + }); + + it('maps the legacy store path onto the JSON adapter', () => { + process.env.OPENCONTEXT_STORE_PATH = '/custom/contexts.json'; + const resolved = resolveDatabase(); + expect(resolved.source).toBe('legacy-store-path'); + expect(resolved.url).toBe('json:///custom/contexts.json'); + }); + + it('prefers the config file over the legacy path', () => { + process.env.OPENCONTEXT_STORE_PATH = '/custom/contexts.json'; + writeDatabaseUrl('sqlite:///data/oc.db'); + expect(resolveDatabase().source).toBe('config-file'); + }); + + it('lets the environment override the config file', () => { + writeDatabaseUrl('sqlite:///data/oc.db'); + process.env.OPENCONTEXT_DB_URL = 'postgres://localhost:5432/oc'; + const resolved = resolveDatabase(); + expect(resolved.source).toBe('env'); + expect(resolved.url).toBe('postgres://localhost:5432/oc'); + }); + + it('locks the value when it comes from the environment', () => { + process.env.OPENCONTEXT_DB_URL = 'postgres://localhost:5432/oc'; + expect(resolveDatabase().locked).toBe(true); + }); + + it('leaves the value editable when it comes from the config file', () => { + writeDatabaseUrl('sqlite:///data/oc.db'); + expect(resolveDatabase().locked).toBe(false); + }); + + it('redacts credentials in the reported value', () => { + process.env.OPENCONTEXT_DB_URL = 'postgres://user:hunter2@host:5432/oc'; + const resolved = resolveDatabase(); + expect(resolved.redacted).not.toContain('hunter2'); + expect(resolved.redacted).toContain('***'); + }); + }); + + describe('writeDatabaseUrl', () => { + it('persists the url', () => { + writeDatabaseUrl('postgres://localhost:5432/oc'); + expect(readConfig().database?.url).toBe('postgres://localhost:5432/oc'); + }); + + it('writes the file with owner-only permissions', () => { + writeDatabaseUrl('postgres://user:secret@host:5432/oc'); + // Connection strings carry passwords, so the file must not be world-readable. + expect(statSync(getConfigPath()).mode & 0o777).toBe(0o600); + }); + + it('overwrites a previous value', () => { + writeDatabaseUrl('sqlite:///a.db'); + writeDatabaseUrl('sqlite:///b.db'); + expect(readConfig().database?.url).toBe('sqlite:///b.db'); + }); + }); + + describe('clearDatabaseUrl', () => { + it('falls back to the default once cleared', () => { + writeDatabaseUrl('sqlite:///data/oc.db'); + clearDatabaseUrl(); + expect(resolveDatabase().source).toBe('default'); + }); + + it('is a no-op when no config exists', () => { + expect(() => clearDatabaseUrl()).not.toThrow(); + }); + }); + + describe('readConfig', () => { + it('falls back to defaults rather than throwing on a corrupt file', () => { + writeFileSync(getConfigPath(), '{ not valid json'); + expect(readConfig().database).toBeUndefined(); + expect(resolveDatabase().source).toBe('default'); + }); + }); +}); diff --git a/tests/store/conformance.ts b/tests/store/conformance.ts new file mode 100644 index 0000000..89ea918 --- /dev/null +++ b/tests/store/conformance.ts @@ -0,0 +1,392 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import type { ContextStoreAdapter } from '../../src/store/types.js'; + +export interface ConformanceHarness { + /** Allocate fresh, empty storage. Called once before each test. */ + setup(): Promise; + /** Open an adapter on the storage `setup` allocated. May be called twice in + * one test to prove data survives reconnection. */ + create(): Promise; + /** Release the storage `setup` allocated. Called once after each test. */ + teardown(): Promise; +} + +/** + * The storage contract, run against every adapter. + * + * Any backend that passes this suite is a drop-in replacement for the others. + * Ordering assertions encode the documented contract: createdAt ascending, then + * id ascending. + */ +export function runStoreConformance(name: string, harness: ConformanceHarness): void { + describe(`${name} — store conformance`, () => { + let store: ContextStoreAdapter; + + beforeEach(async () => { + await harness.setup(); + store = await harness.create(); + }); + + afterEach(async () => { + await store.close(); + await harness.teardown(); + }); + + // ----------------------------------------------------------------------- + // Contexts + // ----------------------------------------------------------------------- + + describe('saveContext', () => { + it('saves an entry and returns it with an id and timestamps', async () => { + const entry = await store.saveContext('My favorite color is blue'); + expect(entry.id).toBeTruthy(); + expect(entry.content).toBe('My favorite color is blue'); + expect(entry.tags).toEqual([]); + expect(entry.source).toBe('chat'); + expect(entry.createdAt).toBeTruthy(); + expect(entry.updatedAt).toBe(entry.createdAt); + }); + + it('stores tags and source', async () => { + const entry = await store.saveContext('Use tabs', ['style', 'code'], 'code-review'); + expect(entry.tags).toEqual(['style', 'code']); + expect(entry.source).toBe('code-review'); + }); + + it('persists across adapter instances', async () => { + await store.saveContext('Persisted entry'); + const reopened = await harness.create(); + try { + const all = await reopened.listContexts(); + expect(all.map((e) => e.content)).toContain('Persisted entry'); + } finally { + await reopened.close(); + } + }); + + it('associates an entry with a bubble', async () => { + const bubble = await store.createBubble('Work'); + const entry = await store.saveContext('Standup at 9', [], 'chat', bubble.id); + expect(entry.bubbleId).toBe(bubble.id); + }); + + it('leaves bubbleId undefined when none is given', async () => { + const entry = await store.saveContext('Unfiled'); + expect(entry.bubbleId).toBeUndefined(); + }); + + it('round-trips content containing quotes and newlines', async () => { + const tricky = `He said "hello"\nthen left; DROP TABLE oc_contexts; --`; + const saved = await store.saveContext(tricky, ["it's"]); + const found = await store.getContext(saved.id); + expect(found?.content).toBe(tricky); + expect(found?.tags).toEqual(["it's"]); + }); + + it('round-trips unicode content', async () => { + const saved = await store.saveContext('日本語 🎉 café'); + expect((await store.getContext(saved.id))?.content).toBe('日本語 🎉 café'); + }); + }); + + describe('getContext', () => { + it('returns the entry by id', async () => { + const saved = await store.saveContext('Find me'); + expect((await store.getContext(saved.id))?.content).toBe('Find me'); + }); + + it('returns undefined for an unknown id', async () => { + expect(await store.getContext('does-not-exist')).toBeUndefined(); + }); + }); + + describe('recallContext', () => { + beforeEach(async () => { + await store.saveContext('I prefer dark mode', ['tooling']); + await store.saveContext('My cat is named Luna'); + await store.saveContext('Dark themes are better for my eyes'); + }); + + it('matches content case-insensitively', async () => { + const results = await store.recallContext('dark'); + expect(results).toHaveLength(2); + }); + + it('matches tags', async () => { + const results = await store.recallContext('tooling'); + expect(results).toHaveLength(1); + expect(results[0]!.content).toBe('I prefer dark mode'); + }); + + it('returns an empty array when nothing matches', async () => { + expect(await store.recallContext('xyz-not-found')).toEqual([]); + }); + + it('is case-insensitive on the query itself', async () => { + expect(await store.recallContext('LUNA')).toHaveLength(1); + }); + }); + + describe('searchContexts', () => { + beforeEach(async () => { + await store.saveContext('TypeScript strict mode is on', ['lang'], 'code-review'); + await store.saveContext('TypeScript is fine', ['lang']); + await store.saveContext('Python is also fine'); + }); + + it('requires every term to match', async () => { + const results = await store.searchContexts('typescript strict'); + expect(results).toHaveLength(1); + }); + + it('searches across content, tags and source', async () => { + const results = await store.searchContexts('typescript code-review'); + expect(results).toHaveLength(1); + }); + + it('returns everything for an all-whitespace query', async () => { + expect(await store.searchContexts(' ')).toHaveLength(3); + }); + + it('returns an empty array when one term fails to match', async () => { + expect(await store.searchContexts('typescript nonexistent')).toEqual([]); + }); + }); + + describe('listContexts', () => { + it('returns an empty array for a fresh store', async () => { + expect(await store.listContexts()).toEqual([]); + }); + + it('returns every entry', async () => { + await store.saveContext('One'); + await store.saveContext('Two'); + expect(await store.listContexts()).toHaveLength(2); + }); + + it('filters by exact tag, case-insensitively', async () => { + await store.saveContext('Tagged', ['Work']); + await store.saveContext('Untagged'); + expect(await store.listContexts('work')).toHaveLength(1); + expect(await store.listContexts('WORK')).toHaveLength(1); + }); + + it('does not match a tag by prefix', async () => { + await store.saveContext('Tagged', ['workspace']); + expect(await store.listContexts('work')).toEqual([]); + }); + + it('orders by createdAt then id', async () => { + await store.saveContext('a'); + await store.saveContext('b'); + await store.saveContext('c'); + const all = await store.listContexts(); + const expected = [...all].sort( + (x, y) => x.createdAt.localeCompare(y.createdAt) || x.id.localeCompare(y.id), + ); + expect(all.map((e) => e.id)).toEqual(expected.map((e) => e.id)); + }); + }); + + describe('updateContext', () => { + it('updates content and bumps updatedAt', async () => { + const saved = await store.saveContext('Original'); + await new Promise((resolve) => setTimeout(resolve, 2)); + const updated = await store.updateContext(saved.id, 'Revised'); + expect(updated?.content).toBe('Revised'); + expect(updated?.createdAt).toBe(saved.createdAt); + expect(updated!.updatedAt >= saved.updatedAt).toBe(true); + }); + + it('replaces tags when given', async () => { + const saved = await store.saveContext('Entry', ['old']); + const updated = await store.updateContext(saved.id, 'Entry', ['new', 'newer']); + expect(updated?.tags).toEqual(['new', 'newer']); + }); + + it('leaves tags alone when omitted', async () => { + const saved = await store.saveContext('Entry', ['keep']); + const updated = await store.updateContext(saved.id, 'Changed'); + expect(updated?.tags).toEqual(['keep']); + }); + + it('assigns a bubble', async () => { + const bubble = await store.createBubble('Proj'); + const saved = await store.saveContext('Entry'); + const updated = await store.updateContext(saved.id, 'Entry', undefined, bubble.id); + expect(updated?.bubbleId).toBe(bubble.id); + }); + + it('unassigns a bubble when passed null', async () => { + const bubble = await store.createBubble('Proj'); + const saved = await store.saveContext('Entry', [], 'chat', bubble.id); + const updated = await store.updateContext(saved.id, 'Entry', undefined, null); + expect(updated?.bubbleId).toBeUndefined(); + }); + + it('persists the update', async () => { + const saved = await store.saveContext('Original'); + await store.updateContext(saved.id, 'Revised'); + expect((await store.getContext(saved.id))?.content).toBe('Revised'); + }); + + it('returns undefined for an unknown id', async () => { + expect(await store.updateContext('nope', 'x')).toBeUndefined(); + }); + }); + + describe('deleteContext', () => { + it('deletes and reports true', async () => { + const saved = await store.saveContext('Delete me'); + expect(await store.deleteContext(saved.id)).toBe(true); + expect(await store.getContext(saved.id)).toBeUndefined(); + }); + + it('reports false for an unknown id', async () => { + expect(await store.deleteContext('nope')).toBe(false); + }); + + it('leaves other entries intact', async () => { + const first = await store.saveContext('Keep'); + const second = await store.saveContext('Remove'); + await store.deleteContext(second.id); + const all = await store.listContexts(); + expect(all).toHaveLength(1); + expect(all[0]!.id).toBe(first.id); + }); + }); + + // ----------------------------------------------------------------------- + // Bubbles + // ----------------------------------------------------------------------- + + describe('createBubble', () => { + it('creates a bubble with an id and timestamps', async () => { + const bubble = await store.createBubble('Side project'); + expect(bubble.id).toBeTruthy(); + expect(bubble.name).toBe('Side project'); + expect(bubble.description).toBeUndefined(); + expect(bubble.updatedAt).toBe(bubble.createdAt); + }); + + it('stores a description when given', async () => { + const bubble = await store.createBubble('Work', 'Day job context'); + expect(bubble.description).toBe('Day job context'); + }); + }); + + describe('listBubbles / getBubble', () => { + it('returns an empty array for a fresh store', async () => { + expect(await store.listBubbles()).toEqual([]); + }); + + it('lists created bubbles', async () => { + await store.createBubble('One'); + await store.createBubble('Two'); + expect(await store.listBubbles()).toHaveLength(2); + }); + + it('gets a bubble by id', async () => { + const created = await store.createBubble('Findable'); + expect((await store.getBubble(created.id))?.name).toBe('Findable'); + }); + + it('returns undefined for an unknown bubble id', async () => { + expect(await store.getBubble('nope')).toBeUndefined(); + }); + }); + + describe('listContextsByBubble', () => { + it('returns only that bubble’s contexts', async () => { + const a = await store.createBubble('A'); + const b = await store.createBubble('B'); + await store.saveContext('in a', [], 'chat', a.id); + await store.saveContext('in b', [], 'chat', b.id); + await store.saveContext('unfiled'); + + const inA = await store.listContextsByBubble(a.id); + expect(inA).toHaveLength(1); + expect(inA[0]!.content).toBe('in a'); + }); + + it('returns an empty array for a bubble with no contexts', async () => { + const bubble = await store.createBubble('Empty'); + expect(await store.listContextsByBubble(bubble.id)).toEqual([]); + }); + }); + + describe('updateBubble', () => { + it('renames a bubble', async () => { + const created = await store.createBubble('Before'); + const updated = await store.updateBubble(created.id, 'After'); + expect(updated?.name).toBe('After'); + }); + + it('updates the description when given', async () => { + const created = await store.createBubble('Name', 'old'); + expect((await store.updateBubble(created.id, 'Name', 'new'))?.description).toBe('new'); + }); + + it('leaves the description alone when omitted', async () => { + const created = await store.createBubble('Name', 'keep'); + expect((await store.updateBubble(created.id, 'Renamed'))?.description).toBe('keep'); + }); + + it('returns undefined for an unknown id', async () => { + expect(await store.updateBubble('nope', 'x')).toBeUndefined(); + }); + }); + + describe('deleteBubble', () => { + it('unassigns its contexts by default rather than deleting them', async () => { + const bubble = await store.createBubble('Temp'); + const entry = await store.saveContext('Survives', [], 'chat', bubble.id); + + expect(await store.deleteBubble(bubble.id)).toBe(true); + expect(await store.getBubble(bubble.id)).toBeUndefined(); + + const survivor = await store.getContext(entry.id); + expect(survivor).toBeDefined(); + expect(survivor?.bubbleId).toBeUndefined(); + }); + + it('deletes its contexts when asked', async () => { + const bubble = await store.createBubble('Temp'); + const entry = await store.saveContext('Goes away', [], 'chat', bubble.id); + + expect(await store.deleteBubble(bubble.id, true)).toBe(true); + expect(await store.getContext(entry.id)).toBeUndefined(); + }); + + it('leaves contexts in other bubbles untouched', async () => { + const doomed = await store.createBubble('Doomed'); + const safe = await store.createBubble('Safe'); + await store.saveContext('in doomed', [], 'chat', doomed.id); + const keeper = await store.saveContext('in safe', [], 'chat', safe.id); + + await store.deleteBubble(doomed.id, true); + expect((await store.getContext(keeper.id))?.bubbleId).toBe(safe.id); + }); + + it('reports false for an unknown id', async () => { + expect(await store.deleteBubble('nope')).toBe(false); + }); + }); + + // ----------------------------------------------------------------------- + // Lifecycle + // ----------------------------------------------------------------------- + + describe('lifecycle', () => { + it('reports adapter info with a redacted target', async () => { + expect(store.info.scheme).toBeTruthy(); + expect(store.info.label).toBeTruthy(); + expect(store.info.target).not.toMatch(/hunter2|secret-token/); + }); + + it('pings a live connection without throwing', async () => { + await expect(store.ping()).resolves.toBeUndefined(); + }); + }); + }); +} diff --git a/tests/store/dsn.test.ts b/tests/store/dsn.test.ts new file mode 100644 index 0000000..963105a --- /dev/null +++ b/tests/store/dsn.test.ts @@ -0,0 +1,301 @@ +import { describe, it, expect } from 'vitest'; +import { parseDsn, redactDsn, SUPPORTED_SCHEMES } from '../../src/store/dsn.js'; +import { InvalidDsnError } from '../../src/store/types.js'; + +describe('parseDsn', () => { + describe('json', () => { + it('parses an absolute file path', () => { + const dsn = parseDsn('json:///home/me/.opencontext/contexts.json'); + expect(dsn.scheme).toBe('json'); + expect(dsn.path).toBe('/home/me/.opencontext/contexts.json'); + expect(dsn.remote).toBe(false); + }); + + it('treats a bare path with no scheme as json', () => { + const dsn = parseDsn('/var/data/contexts.json'); + expect(dsn.scheme).toBe('json'); + expect(dsn.path).toBe('/var/data/contexts.json'); + }); + + it('treats a relative path with no scheme as json', () => { + const dsn = parseDsn('./local/contexts.json'); + expect(dsn.scheme).toBe('json'); + expect(dsn.path).toBe('./local/contexts.json'); + }); + }); + + describe('sqlite', () => { + it('parses a file path', () => { + const dsn = parseDsn('sqlite:///data/oc.db'); + expect(dsn.scheme).toBe('sqlite'); + expect(dsn.path).toBe('/data/oc.db'); + expect(dsn.remote).toBe(false); + }); + + it('parses the in-memory form', () => { + const dsn = parseDsn('sqlite::memory:'); + expect(dsn.scheme).toBe('sqlite'); + expect(dsn.path).toBe(':memory:'); + }); + }); + + describe('libsql', () => { + it('parses a remote host and auth token', () => { + const dsn = parseDsn('libsql://db.turso.io?authToken=secret-token'); + expect(dsn.scheme).toBe('libsql'); + expect(dsn.host).toBe('db.turso.io'); + expect(dsn.params.authToken).toBe('secret-token'); + expect(dsn.remote).toBe(true); + }); + }); + + describe('postgres', () => { + it('parses host, port, credentials and database', () => { + const dsn = parseDsn('postgres://alice:hunter2@db.example.com:5432/opencontext'); + expect(dsn.scheme).toBe('postgres'); + expect(dsn.host).toBe('db.example.com'); + expect(dsn.port).toBe(5432); + expect(dsn.username).toBe('alice'); + expect(dsn.password).toBe('hunter2'); + expect(dsn.database).toBe('opencontext'); + expect(dsn.remote).toBe(true); + }); + + it('accepts postgresql:// as an alias', () => { + expect(parseDsn('postgresql://localhost/oc').scheme).toBe('postgres'); + }); + + it('defaults the port to 5432', () => { + expect(parseDsn('postgres://localhost/oc').port).toBe(5432); + }); + + it('rejects a postgres url with no database', () => { + expect(() => parseDsn('postgres://localhost')).toThrow(InvalidDsnError); + }); + }); + + describe('duckdb', () => { + it('parses a file path', () => { + const dsn = parseDsn('duckdb:///data/oc.duckdb'); + expect(dsn.scheme).toBe('duckdb'); + expect(dsn.path).toBe('/data/oc.duckdb'); + expect(dsn.remote).toBe(false); + }); + + it('parses the in-memory form', () => { + expect(parseDsn('duckdb::memory:').path).toBe(':memory:'); + }); + }); + + describe('surrealdb', () => { + it('parses credentials, namespace and database', () => { + const dsn = parseDsn('surrealdb://root:root@127.0.0.1:8000/myns/mydb'); + expect(dsn.scheme).toBe('surrealdb'); + expect(dsn.host).toBe('127.0.0.1'); + expect(dsn.port).toBe(8000); + expect(dsn.username).toBe('root'); + expect(dsn.password).toBe('root'); + expect(dsn.namespace).toBe('myns'); + expect(dsn.database).toBe('mydb'); + expect(dsn.remote).toBe(true); + }); + + it('accepts ws:// and wss:// aliases', () => { + expect(parseDsn('ws://root:root@localhost:8000/ns/db').scheme).toBe('surrealdb'); + expect(parseDsn('wss://root:root@localhost:8000/ns/db').scheme).toBe('surrealdb'); + }); + + it('builds an http endpoint for ws and a secure one for wss', () => { + expect(parseDsn('ws://localhost:8000/ns/db').endpoint).toBe('http://localhost:8000'); + expect(parseDsn('wss://cloud.surreal.io/ns/db').endpoint).toBe('https://cloud.surreal.io'); + }); + + it('rejects a surreal url missing the database segment', () => { + expect(() => parseDsn('surrealdb://localhost:8000/onlyns')).toThrow(InvalidDsnError); + }); + }); + + describe('mssql / azure sql', () => { + it('parses host, credentials and database', () => { + const dsn = parseDsn('mssql://sa:Secret1@sql.example.com:1433/opencontext'); + expect(dsn.scheme).toBe('mssql'); + expect(dsn.host).toBe('sql.example.com'); + expect(dsn.port).toBe(1433); + expect(dsn.username).toBe('sa'); + expect(dsn.password).toBe('Secret1'); + expect(dsn.database).toBe('opencontext'); + expect(dsn.remote).toBe(true); + }); + + it('defaults the port to 1433', () => { + expect(parseDsn('mssql://host/db').port).toBe(1433); + }); + + it('accepts sqlserver:// and azuresql:// aliases', () => { + expect(parseDsn('sqlserver://host/db').scheme).toBe('mssql'); + expect(parseDsn('azuresql://host/db').scheme).toBe('mssql'); + }); + + it('decodes a percent-encoded Azure username containing @', () => { + const dsn = parseDsn('mssql://admin%40myserver:pw@myserver.database.windows.net/oc'); + expect(dsn.username).toBe('admin@myserver'); + }); + + it('carries the encrypt flag through as a param', () => { + expect(parseDsn('mssql://host/db?encrypt=true').params.encrypt).toBe('true'); + }); + + it('rejects an mssql url with no database', () => { + expect(() => parseDsn('mssql://host')).toThrow(InvalidDsnError); + }); + }); + + describe('cloudsql', () => { + it('parses the instance connection name and database', () => { + const dsn = parseDsn('cloudsql://app:pw@my-proj:us-central1:my-inst/opencontext'); + expect(dsn.scheme).toBe('cloudsql'); + expect(dsn.instance).toBe('my-proj:us-central1:my-inst'); + expect(dsn.database).toBe('opencontext'); + expect(dsn.username).toBe('app'); + expect(dsn.password).toBe('pw'); + expect(dsn.remote).toBe(true); + }); + + it('allows credentials to be omitted for IAM auth', () => { + const dsn = parseDsn('cloudsql://my-proj:us-central1:my-inst/opencontext'); + expect(dsn.instance).toBe('my-proj:us-central1:my-inst'); + expect(dsn.username).toBeUndefined(); + }); + + it('carries query params through', () => { + const dsn = parseDsn('cloudsql://p:r:i/db?ipType=PRIVATE'); + expect(dsn.params.ipType).toBe('PRIVATE'); + }); + + it('rejects an instance name that is not project:region:instance', () => { + expect(() => parseDsn('cloudsql://just-an-instance/db')).toThrow(/project:region:instance/); + expect(() => parseDsn('cloudsql://proj:region/db')).toThrow(/project:region:instance/); + }); + + it('rejects a cloudsql url with no database', () => { + expect(() => parseDsn('cloudsql://p:r:i')).toThrow(InvalidDsnError); + }); + }); + + describe('dynamodb', () => { + it('parses region and table', () => { + const dsn = parseDsn('dynamodb://us-east-1/opencontext'); + expect(dsn.scheme).toBe('dynamodb'); + expect(dsn.region).toBe('us-east-1'); + expect(dsn.table).toBe('opencontext'); + expect(dsn.remote).toBe(true); + }); + + it('accepts the ddb:// alias', () => { + expect(parseDsn('ddb://eu-west-2/tbl').scheme).toBe('dynamodb'); + }); + + it('carries a local endpoint override through as a param', () => { + const dsn = parseDsn('dynamodb://us-east-1/oc?endpoint=http://localhost:8000'); + expect(dsn.params.endpoint).toBe('http://localhost:8000'); + }); + + it('rejects a dynamodb url with no table', () => { + expect(() => parseDsn('dynamodb://us-east-1')).toThrow(InvalidDsnError); + }); + }); + + describe('validation', () => { + it('rejects an unknown scheme and names the supported ones', () => { + expect(() => parseDsn('cassandra://localhost/oc')).toThrow(/Unsupported/); + try { + parseDsn('cassandra://localhost/oc'); + } catch (error) { + expect((error as Error).message).toContain('postgres'); + expect((error as Error).message).toContain('mongodb'); + } + }); + + it('rejects an empty connection string', () => { + expect(() => parseDsn('')).toThrow(InvalidDsnError); + expect(() => parseDsn(' ')).toThrow(InvalidDsnError); + }); + + it('rejects a file-based scheme with no path', () => { + expect(() => parseDsn('sqlite://')).toThrow(InvalidDsnError); + }); + + it('lists every supported scheme', () => { + expect(SUPPORTED_SCHEMES).toEqual([ + 'json', 'memory', 'sqlite', 'duckdb', + 'libsql', 'd1', 'postgres', 'cloudsql', 'mysql', 'mssql', + 'mongodb', 'redis', 'firestore', 'dynamodb', 'surrealdb', + ]); + }); + }); +}); + +describe('canonical connection string', () => { + it('rewrites an alias to the spelling the driver library accepts', () => { + expect(parseDsn('mongo://host:27017/oc').canonical).toBe('mongodb://host:27017/oc'); + expect(parseDsn('valkey://host:6379').canonical).toBe('redis://host:6379'); + expect(parseDsn('postgresql://host/oc').canonical).toBe('postgres://host/oc'); + }); + + it('lowercases a shouted scheme', () => { + expect(parseDsn('MONGODB://host:27017/oc').canonical).toBe('mongodb://host:27017/oc'); + }); + + it('preserves rediss:// so TLS is not silently downgraded', () => { + // rediss is the TLS variant of redis. Collapsing it to the normalised + // scheme would turn encryption off without telling anyone. + expect(parseDsn('rediss://host:6379').canonical).toBe('rediss://host:6379'); + }); + + it('preserves mongodb+srv:// so Atlas SRV lookup still happens', () => { + expect(parseDsn('mongodb+srv://cluster.mongodb.net/oc').canonical).toBe( + 'mongodb+srv://cluster.mongodb.net/oc', + ); + }); + + it('leaves credentials and query params intact', () => { + expect(parseDsn('mongo://u:p@host:27017/oc?retryWrites=true').canonical).toBe( + 'mongodb://u:p@host:27017/oc?retryWrites=true', + ); + }); + + it('leaves schemes with no alias untouched', () => { + expect(parseDsn('mongodb://host/oc').canonical).toBe('mongodb://host/oc'); + expect(parseDsn('redis://host:6379').canonical).toBe('redis://host:6379'); + }); + + it('is set for file-based schemes too', () => { + expect(parseDsn('sqlite:///data/oc.db').canonical).toBe('sqlite:///data/oc.db'); + expect(parseDsn('/data/contexts.json').canonical).toBe('/data/contexts.json'); + }); +}); + +describe('redactDsn', () => { + it('masks the password', () => { + expect(redactDsn('postgres://alice:hunter2@db.example.com:5432/oc')) + .toBe('postgres://alice:***@db.example.com:5432/oc'); + }); + + it('masks an auth token query parameter', () => { + expect(redactDsn('libsql://db.turso.io?authToken=secret')) + .toBe('libsql://db.turso.io?authToken=***'); + }); + + it('leaves a url with no credentials untouched', () => { + expect(redactDsn('postgres://localhost:5432/oc')).toBe('postgres://localhost:5432/oc'); + }); + + it('leaves file paths untouched', () => { + expect(redactDsn('sqlite:///data/oc.db')).toBe('sqlite:///data/oc.db'); + expect(redactDsn('/data/contexts.json')).toBe('/data/contexts.json'); + }); + + it('returns unparseable input unchanged rather than throwing', () => { + expect(redactDsn('not a url at all')).toBe('not a url at all'); + }); +}); diff --git a/tests/store/json-adapter.test.ts b/tests/store/json-adapter.test.ts new file mode 100644 index 0000000..2cdd8a4 --- /dev/null +++ b/tests/store/json-adapter.test.ts @@ -0,0 +1,79 @@ +import { rmSync, mkdirSync, writeFileSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { randomUUID } from 'crypto'; +import { createJsonAdapter } from '../../src/store/adapters/json.js'; +import { parseDsn } from '../../src/store/dsn.js'; +import { runStoreConformance } from './conformance.js'; +import { describe, it, expect, afterEach } from 'vitest'; +import { existsSync } from 'fs'; + +let dir: string; + +runStoreConformance('json', { + async setup() { + dir = join(tmpdir(), `opencontext-json-${randomUUID()}`); + mkdirSync(dir, { recursive: true }); + }, + async create() { + const adapter = createJsonAdapter(parseDsn(join(dir, 'contexts.json'))); + await adapter.connect(); + return adapter; + }, + async teardown() { + rmSync(dir, { recursive: true, force: true }); + }, +}); + +// --------------------------------------------------------------------------- +// Behaviour specific to the file-backed adapter, carried over from the original +// store tests that the conformance suite does not cover. +// --------------------------------------------------------------------------- + +describe('json adapter — file handling', () => { + const created: string[] = []; + + afterEach(() => { + for (const path of created.splice(0)) { + rmSync(path, { recursive: true, force: true }); + } + }); + + it('creates parent directories that do not exist yet', async () => { + const root = join(tmpdir(), `opencontext-json-nested-${randomUUID()}`); + created.push(root); + const file = join(root, 'deeply', 'nested', 'contexts.json'); + + const adapter = createJsonAdapter(parseDsn(file)); + await adapter.connect(); + await adapter.saveContext('needs a directory'); + await adapter.close(); + + expect(existsSync(file)).toBe(true); + }); + + it('treats a missing store file as an empty store rather than an error', async () => { + const root = join(tmpdir(), `opencontext-json-missing-${randomUUID()}`); + created.push(root); + + const adapter = createJsonAdapter(parseDsn(join(root, 'contexts.json'))); + await adapter.connect(); + expect(await adapter.listContexts()).toEqual([]); + expect(await adapter.listBubbles()).toEqual([]); + await adapter.close(); + }); + + it('migrates a store written before bubbles existed', async () => { + const root = join(tmpdir(), `opencontext-json-legacy-${randomUUID()}`); + created.push(root); + mkdirSync(root, { recursive: true }); + const file = join(root, 'contexts.json'); + // A v1 store had no `bubbles` key at all. + writeFileSync(file, JSON.stringify({ version: 1, entries: [] }), 'utf-8'); + + const adapter = createJsonAdapter(parseDsn(file)); + await adapter.connect(); + expect(await adapter.listBubbles()).toEqual([]); + await adapter.close(); + }); +}); diff --git a/tests/store/migrate.test.ts b/tests/store/migrate.test.ts new file mode 100644 index 0000000..b0d5d8a --- /dev/null +++ b/tests/store/migrate.test.ts @@ -0,0 +1,109 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { rmSync, mkdirSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { randomUUID } from 'crypto'; +import { createStore } from '../../src/store/index.js'; +import { migrateStore } from '../../src/store/migrate.js'; +import type { ContextStoreAdapter } from '../../src/store/types.js'; + +describe('migrateStore', () => { + let dir: string; + let source: ContextStoreAdapter; + let target: ContextStoreAdapter; + + beforeEach(async () => { + dir = join(tmpdir(), `opencontext-migrate-${randomUUID()}`); + mkdirSync(dir, { recursive: true }); + // JSON to SQLite — the migration people actually run when they outgrow the + // default file store. + source = await createStore(`json://${join(dir, 'contexts.json')}`); + target = await createStore(`sqlite://${join(dir, 'oc.db')}`); + }); + + afterEach(async () => { + await source.close(); + await target.close(); + rmSync(dir, { recursive: true, force: true }); + }); + + it('copies contexts and bubbles across', async () => { + const bubble = await source.createBubble('Work', 'Day job'); + await source.saveContext('Standup at 9', ['meeting'], 'chat', bubble.id); + await source.saveContext('Unfiled note'); + + const result = await migrateStore(source, target); + + expect(result).toEqual({ contexts: 2, bubbles: 1 }); + expect(await target.listContexts()).toHaveLength(2); + expect(await target.listBubbles()).toHaveLength(1); + }); + + it('preserves the context to bubble relationship', async () => { + const bubble = await source.createBubble('Work'); + await source.saveContext('In a bubble', [], 'chat', bubble.id); + + await migrateStore(source, target); + + const targetBubble = (await target.listBubbles())[0]!; + const inBubble = await target.listContextsByBubble(targetBubble.id); + expect(inBubble).toHaveLength(1); + expect(inBubble[0]!.content).toBe('In a bubble'); + }); + + it('preserves tags and source', async () => { + await source.saveContext('Tagged', ['a', 'b'], 'code-review'); + + await migrateStore(source, target); + + const entry = (await target.listContexts())[0]!; + expect(entry.tags).toEqual(['a', 'b']); + expect(entry.source).toBe('code-review'); + }); + + it('adds to existing data in copy mode', async () => { + await target.saveContext('Already there'); + await source.saveContext('Incoming'); + + await migrateStore(source, target, { mode: 'copy' }); + + expect(await target.listContexts()).toHaveLength(2); + }); + + it('empties the target first in replace mode', async () => { + await target.saveContext('Should be gone'); + await source.saveContext('Incoming'); + + await migrateStore(source, target, { mode: 'replace' }); + + const remaining = await target.listContexts(); + expect(remaining).toHaveLength(1); + expect(remaining[0]!.content).toBe('Incoming'); + }); + + it('never mutates the source', async () => { + await source.saveContext('Original'); + const before = await source.listContexts(); + + await migrateStore(source, target, { mode: 'replace' }); + + expect(await source.listContexts()).toEqual(before); + }); + + it('survives replace mode when source and target are the same store', async () => { + // Reading before clearing is what makes this safe — clearing first would + // delete the very rows about to be copied. + await source.saveContext('do not lose me', ['important']); + const sameStore = await createStore(`json://${join(dir, 'contexts.json')}`); + + const result = await migrateStore(source, sameStore, { mode: 'replace' }); + + expect(result.contexts).toBe(1); + expect((await sameStore.listContexts())[0]!.content).toBe('do not lose me'); + await sameStore.close(); + }); + + it('handles an empty source', async () => { + expect(await migrateStore(source, target)).toEqual({ contexts: 0, bubbles: 0 }); + }); +}); diff --git a/tests/store/sqlite-adapter.test.ts b/tests/store/sqlite-adapter.test.ts new file mode 100644 index 0000000..b844f44 --- /dev/null +++ b/tests/store/sqlite-adapter.test.ts @@ -0,0 +1,34 @@ +import { rmSync, mkdirSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { randomUUID } from 'crypto'; +import { createSqliteDriver } from '../../src/store/drivers/sqlite.js'; +import { createSqlAdapter } from '../../src/store/adapters/sql.js'; +import { parseDsn } from '../../src/store/dsn.js'; +import { runStoreConformance } from './conformance.js'; + +let dir: string; + +// SQLite runs unconditionally: `node:sqlite` is built into Node, so this gives +// the shared SQL adapter real coverage with no external service. +runStoreConformance('sqlite', { + async setup() { + dir = join(tmpdir(), `opencontext-sqlite-${randomUUID()}`); + mkdirSync(dir, { recursive: true }); + }, + async create() { + const dsn = parseDsn(`sqlite://${join(dir, 'oc.db')}`); + const driver = await createSqliteDriver(dsn); + const adapter = createSqlAdapter(driver, { + scheme: 'sqlite', + label: 'SQLite', + target: dsn.path!, + remote: false, + }); + await adapter.connect(); + return adapter; + }, + async teardown() { + rmSync(dir, { recursive: true, force: true }); + }, +}); diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 620125b..c67f2fc 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -11,6 +11,7 @@ import VendorExport from './components/VendorExport'; import ContextsManager from './components/ContextsManager'; import BubblesManager from './components/BubblesManager'; import ChatWithContext from './components/ChatWithContext'; +import DatabaseSettings from './components/DatabaseSettings'; import './App.css'; export default function App() { @@ -30,6 +31,7 @@ export default function App() { } /> } /> } /> + } /> } /> diff --git a/ui/src/components/DatabaseSettings.tsx b/ui/src/components/DatabaseSettings.tsx new file mode 100644 index 0000000..ce35951 --- /dev/null +++ b/ui/src/components/DatabaseSettings.tsx @@ -0,0 +1,353 @@ +import { useState, useEffect, useCallback } from 'react'; +import { Input } from '@/components/ui/input'; +import { Button } from '@/components/ui/button'; +import { Badge } from '@/components/ui/badge'; +import { Card, CardContent, CardHeader } from '@/components/ui/card'; +import { Label } from '@/components/ui/label'; +import { + Database, + Check, + X, + Loader2, + Cloud, + HardDrive, + Lock, + ArrowRightLeft, + AlertTriangle, +} from 'lucide-react'; + +// --------------------------------------------------------------------------- +// Types mirroring the /api/db responses +// --------------------------------------------------------------------------- + +interface AdapterInfo { + scheme: string; + label: string; + target: string; + remote: boolean; +} + +interface AdapterOption { + scheme: string; + label: string; + example: string; + packageName: string | null; + remote: boolean; + family: 'file' | 'sql' | 'document'; + installed: boolean; +} + +interface DbStatus { + connected: boolean; + adapter: AdapterInfo | null; + source: 'env' | 'config-file' | 'legacy-store-path' | 'default'; + locked: boolean; + url: string; + counts: { contexts: number; bubbles: number } | null; + error?: string; +} + +const SOURCE_LABELS: Record = { + env: 'OPENCONTEXT_DB_URL environment variable', + 'config-file': 'saved settings', + 'legacy-store-path': 'OPENCONTEXT_STORE_PATH environment variable', + default: 'default (no configuration)', +}; + +const FAMILY_LABELS: Record = { + file: 'Local file', + sql: 'SQL', + document: 'Document & key-value', +}; + +type Feedback = { kind: 'ok' | 'error'; message: string } | null; + +export default function DatabaseSettings() { + const [status, setStatus] = useState(null); + const [adapters, setAdapters] = useState([]); + const [url, setUrl] = useState(''); + const [testing, setTesting] = useState(false); + const [saving, setSaving] = useState(false); + const [migrating, setMigrating] = useState(false); + const [testResult, setTestResult] = useState(null); + const [saveResult, setSaveResult] = useState(null); + const [migrateResult, setMigrateResult] = useState(null); + + const loadStatus = useCallback(async () => { + const response = await fetch('/api/db/status'); + setStatus((await response.json()) as DbStatus); + }, []); + + useEffect(() => { + void loadStatus(); + void fetch('/api/db/adapters') + .then((r) => r.json()) + .then((data) => setAdapters(data as AdapterOption[])); + }, [loadStatus]); + + async function handleTest() { + setTesting(true); + setTestResult(null); + try { + const response = await fetch('/api/db/test', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url }), + }); + const data = await response.json(); + setTestResult( + data.ok + ? { kind: 'ok', message: `Connected to ${data.adapter.label}` } + : { kind: 'error', message: data.error }, + ); + } catch (error) { + setTestResult({ kind: 'error', message: (error as Error).message }); + } finally { + setTesting(false); + } + } + + async function handleSave() { + setSaving(true); + setSaveResult(null); + try { + const response = await fetch('/api/db/config', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url }), + }); + const data = await response.json(); + if (data.ok) { + setSaveResult({ kind: 'ok', message: `Now using ${data.adapter.label}` }); + setUrl(''); + await loadStatus(); + } else { + setSaveResult({ kind: 'error', message: data.error }); + } + } catch (error) { + setSaveResult({ kind: 'error', message: (error as Error).message }); + } finally { + setSaving(false); + } + } + + async function handleMigrate() { + setMigrating(true); + setMigrateResult(null); + try { + const response = await fetch('/api/db/migrate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url, mode: 'copy' }), + }); + const data = await response.json(); + setMigrateResult( + data.ok + ? { + kind: 'ok', + message: `Copied ${data.contexts} contexts and ${data.bubbles} bubbles. Your current store was not changed.`, + } + : { kind: 'error', message: data.error }, + ); + } catch (error) { + setMigrateResult({ kind: 'error', message: (error as Error).message }); + } finally { + setMigrating(false); + } + } + + const selected = adapters.find((adapter) => url.startsWith(`${adapter.scheme}:`)); + const missingDriver = selected && !selected.installed ? selected : null; + + return ( +
+
+

Database

+

+ Store your contexts wherever you like — a local file, an embedded database, or your + own server. Everything stays on infrastructure you control. +

+
+ + {/* ---------------- current backend ---------------- */} + + +
+ + Current store + {status?.connected ? ( + + Connected + + ) : status ? ( + + Not connected + + ) : null} +
+ + + {!status ? ( +

Loading…

+ ) : ( + <> +
+ {status.adapter?.remote ? ( + + ) : ( + + )} + + {status.adapter?.label ?? 'Unknown'} + +
+ +
+ {status.url} +
+ +
+ Configured by {SOURCE_LABELS[status.source]} +
+ + {status.counts && ( +
+ {status.counts.contexts} contexts · {status.counts.bubbles} bubbles +
+ )} + + {status.error && ( +
+ + {status.error} +
+ )} + + {status.locked && ( +
+ + + An environment variable is setting the database, so it takes precedence over + anything saved here. Unset it to change the store from this page. + +
+ )} + + )} + + + + {/* ---------------- pick a backend ---------------- */} + + + Connect a different database + + +
+ {(['file', 'sql', 'document'] as const).map((family) => ( +
+
{FAMILY_LABELS[family]}
+
+ {adapters + .filter((adapter) => adapter.family === family) + .map((adapter) => ( + + ))} +
+
+ ))} +
+ +
+ + setUrl(event.target.value)} + placeholder="postgres://user:password@host:5432/opencontext" + className="font-mono text-xs" + autoComplete="off" + spellCheck={false} + /> +

+ Saved to ~/.opencontext/config.json with + owner-only permissions. It never leaves this machine. +

+
+ + {missingDriver && ( +
+ + + {missingDriver.label} needs a driver that is not installed yet. Run{' '} + + npm install {missingDriver.packageName} + + . + +
+ )} + +
+ + + +
+ + {[testResult, saveResult, migrateResult].map( + (result, index) => + result && ( +
+ {result.kind === 'ok' ? ( + + ) : ( + + )} + {result.message} +
+ ), + )} + + +
+ ); +} diff --git a/ui/src/components/Layout.tsx b/ui/src/components/Layout.tsx index ddfa113..9bbfdc7 100644 --- a/ui/src/components/Layout.tsx +++ b/ui/src/components/Layout.tsx @@ -1,5 +1,5 @@ import { NavLink, Outlet, useNavigate } from 'react-router-dom'; -import { LayoutDashboard, Settings, MessageSquare, GitBranch, Download, LogOut, Brain, Layers, MessageCircle } from 'lucide-react'; +import { LayoutDashboard, Settings, MessageSquare, GitBranch, Download, LogOut, Brain, Layers, MessageCircle, Database } from 'lucide-react'; import { useAuth } from '../store/auth'; import { Button } from '@/components/ui/button'; @@ -12,6 +12,7 @@ const navItems = [ { to: '/contexts', icon: Brain, label: 'Contexts' }, { to: '/bubbles', icon: Layers, label: 'Bubbles' }, { to: '/chat', icon: MessageCircle, label: 'Start Chat' }, + { to: '/database', icon: Database, label: 'Database' }, ]; export default function Layout() { diff --git a/ui/src/components/__tests__/DatabaseSettings.test.tsx b/ui/src/components/__tests__/DatabaseSettings.test.tsx new file mode 100644 index 0000000..03a04b8 --- /dev/null +++ b/ui/src/components/__tests__/DatabaseSettings.test.tsx @@ -0,0 +1,207 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor, fireEvent } from '@testing-library/react'; +import React from 'react'; +import DatabaseSettings from '../DatabaseSettings'; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const adapters = [ + { + scheme: 'json', + label: 'JSON file', + example: 'json:///path/to/contexts.json', + packageName: null, + remote: false, + family: 'file', + installed: true, + }, + { + scheme: 'postgres', + label: 'PostgreSQL', + example: 'postgres://USER:PASSWORD@HOST:5432/DATABASE', + packageName: 'pg', + remote: true, + family: 'sql', + installed: true, + }, + { + scheme: 'firestore', + label: 'Google Firestore', + example: 'firestore://PROJECT_ID', + packageName: '@google-cloud/firestore', + remote: true, + family: 'document', + installed: false, + }, +]; + +function statusFixture(overrides: Record = {}) { + return { + connected: true, + adapter: { scheme: 'json', label: 'JSON file', target: '/home/me/contexts.json', remote: false }, + source: 'default', + locked: false, + url: 'json:///home/me/contexts.json', + counts: { contexts: 12, bubbles: 3 }, + ...overrides, + }; +} + +/** Route each endpoint to a canned response; extra handlers override defaults. */ +function mockFetch(handlers: Record = {}) { + return vi.fn(async (url: string, init?: RequestInit) => { + const key = `${init?.method ?? 'GET'} ${url}`; + const body = + key in handlers + ? handlers[key] + : url === '/api/db/adapters' + ? adapters + : url === '/api/db/status' + ? statusFixture() + : {}; + return { ok: true, json: async () => body } as Response; + }); +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe('DatabaseSettings', () => { + it('shows the current backend and its contents', async () => { + global.fetch = mockFetch() as unknown as typeof fetch; + render(); + + // "JSON file" appears twice — as the current store and as a pickable + // adapter — so assert on the values unique to the status card. + expect(await screen.findByText('json:///home/me/contexts.json')).toBeInTheDocument(); + expect(screen.getAllByText('JSON file').length).toBeGreaterThan(0); + expect(screen.getByText(/12 contexts/)).toBeInTheDocument(); + expect(screen.getByText('Connected')).toBeInTheDocument(); + }); + + it('reports a backend it could not reach, with the error', async () => { + global.fetch = mockFetch({ + 'GET /api/db/status': statusFixture({ + connected: false, + adapter: null, + counts: null, + error: 'connection refused', + }), + }) as unknown as typeof fetch; + + render(); + + expect(await screen.findByText('Not connected')).toBeInTheDocument(); + expect(screen.getByText('connection refused')).toBeInTheDocument(); + }); + + it('lists every supported adapter grouped by family', async () => { + global.fetch = mockFetch() as unknown as typeof fetch; + render(); + + expect(await screen.findByText('PostgreSQL')).toBeInTheDocument(); + expect(screen.getByText('Google Firestore')).toBeInTheDocument(); + expect(screen.getByText('SQL')).toBeInTheDocument(); + expect(screen.getByText('Document & key-value')).toBeInTheDocument(); + }); + + it('fills the connection field from the adapter template when one is picked', async () => { + global.fetch = mockFetch() as unknown as typeof fetch; + render(); + + fireEvent.click(await screen.findByText('PostgreSQL')); + + expect(screen.getByLabelText(/connection string/i)).toHaveValue( + 'postgres://USER:PASSWORD@HOST:5432/DATABASE', + ); + }); + + it('tells the user what to install when a driver is missing', async () => { + global.fetch = mockFetch() as unknown as typeof fetch; + render(); + + fireEvent.click(await screen.findByText('Google Firestore')); + + expect(await screen.findByText(/npm install @google-cloud\/firestore/)).toBeInTheDocument(); + }); + + it('reports a successful connection test', async () => { + global.fetch = mockFetch({ + 'POST /api/db/test': { ok: true, adapter: { label: 'PostgreSQL' } }, + }) as unknown as typeof fetch; + render(); + + fireEvent.click(await screen.findByText('PostgreSQL')); + fireEvent.click(screen.getByRole('button', { name: /test connection/i })); + + expect(await screen.findByText('Connected to PostgreSQL')).toBeInTheDocument(); + }); + + it('surfaces the driver error when a connection test fails', async () => { + global.fetch = mockFetch({ + 'POST /api/db/test': { ok: false, error: 'password authentication failed' }, + }) as unknown as typeof fetch; + render(); + + fireEvent.click(await screen.findByText('PostgreSQL')); + fireEvent.click(screen.getByRole('button', { name: /test connection/i })); + + expect(await screen.findByText('password authentication failed')).toBeInTheDocument(); + }); + + it('saves a new connection and refreshes the status', async () => { + const fetchMock = mockFetch({ + 'PUT /api/db/config': { ok: true, adapter: { label: 'PostgreSQL' } }, + }); + global.fetch = fetchMock as unknown as typeof fetch; + render(); + + fireEvent.click(await screen.findByText('PostgreSQL')); + fireEvent.click(screen.getByRole('button', { name: /save & switch/i })); + + expect(await screen.findByText('Now using PostgreSQL')).toBeInTheDocument(); + await waitFor(() => { + expect(fetchMock).toHaveBeenCalledWith('/api/db/status'); + }); + }); + + it('reports what a migration copied and that the source was untouched', async () => { + global.fetch = mockFetch({ + 'POST /api/db/migrate': { ok: true, contexts: 12, bubbles: 3 }, + }) as unknown as typeof fetch; + render(); + + fireEvent.click(await screen.findByText('PostgreSQL')); + fireEvent.click(screen.getByRole('button', { name: /copy my data here/i })); + + expect( + await screen.findByText(/Copied 12 contexts and 3 bubbles/), + ).toBeInTheDocument(); + expect(screen.getByText(/current store was not changed/)).toBeInTheDocument(); + }); + + it('explains and disables saving when an environment variable pins the database', async () => { + global.fetch = mockFetch({ + 'GET /api/db/status': statusFixture({ source: 'env', locked: true }), + }) as unknown as typeof fetch; + render(); + + expect(await screen.findByText(/environment variable is setting the database/i)) + .toBeInTheDocument(); + + fireEvent.click(screen.getByText('PostgreSQL')); + expect(screen.getByRole('button', { name: /save & switch/i })).toBeDisabled(); + }); + + it('keeps the action buttons disabled until a connection string is entered', async () => { + global.fetch = mockFetch() as unknown as typeof fetch; + render(); + + await screen.findByText('PostgreSQL'); + expect(screen.getByRole('button', { name: /test connection/i })).toBeDisabled(); + expect(screen.getByRole('button', { name: /copy my data here/i })).toBeDisabled(); + }); +}); From e67a3ab2784a487a98a8e9af41d124ae242cdac8 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 17 Aug 2026 21:19:19 -0500 Subject: [PATCH 4/8] refactor(store): unify Redis client teardown behind abandon() Dropping a client without waiting for the server was duplicated between shutdown() and a failed connect(). Both paths need it for the same reason: a half-open client keeps a socket and a reconnect timer, and those keep the process alive after the caller has finished with the store. Also falls back to disconnect() on node-redis v4, where destroy() does not exist yet, so the driver tears down cleanly on both major versions. Claude-Session: https://claude.ai/code/session_01CrC2FRcZVEti9kGyhsxaip --- src/store/drivers/redis.ts | 43 +++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/store/drivers/redis.ts b/src/store/drivers/redis.ts index f9bf874..d2d1cae 100644 --- a/src/store/drivers/redis.ts +++ b/src/store/drivers/redis.ts @@ -104,7 +104,7 @@ export async function createRedisDriver(dsn: ParsedDsn): Promise const client = createClient({ url, socket: { - reconnectStrategy(retries: number, cause: Error): number | false { + reconnectStrategy(retries: number): number | false { // node-redis retries the *first* connection forever by default, so a // typo in the host or a Redis that is not running would hang // `createStore` rather than fail it. Nothing about a URL that has never @@ -137,20 +137,34 @@ export async function createRedisDriver(dsn: ParsedDsn): Promise /** Set once the caller has closed the store, so shutdown stays idempotent. */ let closed = false; + /** + * Drop the client without waiting for the server to answer. + * + * A client left half-open keeps a socket and a reconnect timer, and those keep + * the whole process alive long after the caller has finished with the store. + */ + function abandon(): void { + try { + if (client.destroy) { + client.destroy(); + } else { + // What node-redis called the same thing before v5. + void client.disconnect?.().catch(() => {}); + } + } catch { + // Already gone. + } + } + async function shutdown(): Promise { try { // `quit` is deprecated in favour of `close` from node-redis v5 on; both // wait for in-flight commands. await (client.close ? client.close() : client.quit()); } catch { - // The socket was already gone, so there was nobody to answer QUIT. Drop - // the client outright — a shutdown path must not throw, and a client left - // half-open keeps a reconnect timer and the process alive with it. - try { - client.destroy?.(); - } catch { - // Already destroyed. - } + // The socket was already gone, so there was nobody to answer QUIT. A + // shutdown path must not throw, so tear the client down instead. + abandon(); } } @@ -162,14 +176,9 @@ export async function createRedisDriver(dsn: ParsedDsn): Promise try { await client.connect(); } catch (error) { - // A connect that never succeeded still leaves a socket and a retry - // timer behind. Tear them down before reporting, or the process hangs - // long after the caller has given up. - try { - client.destroy?.(); - } catch { - // Nothing was open in the first place. - } + // A connect that never succeeded can still leave a socket and a retry + // timer behind. Tear them down before reporting the failure. + abandon(); closed = true; throw error; } From 1bacd22f64226323ee1ba685bd046fe1b9d42c3f Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 17 Aug 2026 21:19:19 -0500 Subject: [PATCH 5/8] docs(store): expand BYODB README coverage and add backend logos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README described BYODB in its own section but the rest of the document still assumed a JSON file: no db commands, no OPENCONTEXT_DB_URL, no /api/db/* endpoints, and a project tree with no src/store. Brings all of those up to date and records the conformance suite in the dev workflow. Adds a vendor mark for each of the 14 backends that has one, shown in the README table and on the Database settings page. The files are vendored under ui/public/db-logos rather than hotlinked, so the README renders offline and the UI has no third-party request at runtime. Simple Icons no longer ships the Microsoft or Amazon marks, so SQL Server and DynamoDB come from Devicon; both licences are recorded alongside the assets. Three colours are adjusted for legibility — JSON is pure black and invisible in dark mode, DuckDB and Turso too light to read on white. Claude-Session: https://claude.ai/code/session_01CrC2FRcZVEti9kGyhsxaip --- README.md | 95 +++++++++++++++---- ui/public/db-logos/README.md | 45 +++++++++ ui/public/db-logos/cloudsql.svg | 1 + ui/public/db-logos/d1.svg | 1 + ui/public/db-logos/duckdb.svg | 1 + ui/public/db-logos/dynamodb.svg | 1 + ui/public/db-logos/firestore.svg | 1 + ui/public/db-logos/json.svg | 1 + ui/public/db-logos/libsql.svg | 1 + ui/public/db-logos/mongodb.svg | 1 + ui/public/db-logos/mssql.svg | 1 + ui/public/db-logos/mysql.svg | 1 + ui/public/db-logos/postgres.svg | 1 + ui/public/db-logos/redis.svg | 1 + ui/public/db-logos/sqlite.svg | 1 + ui/public/db-logos/surrealdb.svg | 1 + ui/src/components/DatabaseSettings.tsx | 45 ++++++++- .../__tests__/DatabaseSettings.test.tsx | 28 ++++++ 18 files changed, 206 insertions(+), 21 deletions(-) create mode 100644 ui/public/db-logos/README.md create mode 100644 ui/public/db-logos/cloudsql.svg create mode 100644 ui/public/db-logos/d1.svg create mode 100644 ui/public/db-logos/duckdb.svg create mode 100644 ui/public/db-logos/dynamodb.svg create mode 100644 ui/public/db-logos/firestore.svg create mode 100644 ui/public/db-logos/json.svg create mode 100644 ui/public/db-logos/libsql.svg create mode 100644 ui/public/db-logos/mongodb.svg create mode 100644 ui/public/db-logos/mssql.svg create mode 100644 ui/public/db-logos/mysql.svg create mode 100644 ui/public/db-logos/postgres.svg create mode 100644 ui/public/db-logos/redis.svg create mode 100644 ui/public/db-logos/sqlite.svg create mode 100644 ui/public/db-logos/surrealdb.svg diff --git a/README.md b/README.md index f8bdc95..cb20c6a 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ [![Version](https://img.shields.io/badge/version-0.0.1-blue)](https://hub.docker.com/r/adityakarnam/open-context/tags) ![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen) -[Features](#-features) • [Quick Start](#-quick-start) • [Usage](#-usage) • [Documentation](#-documentation) • [Contributing](#-contributing) +[Features](#-features) • [BYODB](#byodb) • [Quick Start](#-quick-start) • [Usage](#-usage) • [Documentation](#-documentation) • [Contributing](#-contributing)
@@ -99,12 +99,18 @@ Switching AI assistants means losing all prior context — your communication st --- + + ## 🗄️ Bring Your Own Database (BYODB) By default opencontext keeps everything in a JSON file at `~/.opencontext/contexts.json` — zero configuration, nothing to install. When you outgrow that, point it at any of **15 backends** without changing how the CLI, the web UI, or the MCP tools behave. +

+ +

+ ```bash # See what is available and what is installed opencontext db adapters @@ -124,21 +130,21 @@ copy your data across — no terminal required. | Backend | Connection string | Install | |---|---|---| -| **JSON file** *(default)* | `json:///path/to/contexts.json` | — built in | +| **JSON file** *(default)* | `json:///path/to/contexts.json` | — built in | | **In-memory** | `memory://` | — built in | -| **SQLite** | `sqlite:///path/to/opencontext.db` | — built in (`node:sqlite`) | -| **Cloudflare D1** | `d1://ACCOUNT_ID/DATABASE_ID?apiToken=TOKEN` | — built in (HTTP) | -| **DuckDB** | `duckdb:///path/to/opencontext.duckdb` | `npm i @duckdb/node-api` | -| **libSQL / Turso** | `libsql://DB.turso.io?authToken=TOKEN` | `npm i @libsql/client` | -| **PostgreSQL** | `postgres://user:pass@host:5432/db` | `npm i pg` | -| **Google Cloud SQL** | `cloudsql://user:pass@PROJECT:REGION:INSTANCE/db` | `npm i @google-cloud/cloud-sql-connector pg` | -| **MySQL / MariaDB** | `mysql://user:pass@host:3306/db` | `npm i mysql2` | -| **SQL Server / Azure SQL** | `mssql://user:pass@host:1433/db` | `npm i mssql` | -| **MongoDB** | `mongodb://user:pass@host:27017/db` | `npm i mongodb` | -| **Redis / Valkey** | `redis://host:6379` | `npm i redis` | -| **Google Firestore** | `firestore://PROJECT_ID` | `npm i @google-cloud/firestore` | -| **Amazon DynamoDB** | `dynamodb://REGION/TABLE` | `npm i @aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb` | -| **SurrealDB** | `surrealdb://user:pass@host:8000/ns/db` | `npm i surrealdb` | +| **SQLite** | `sqlite:///path/to/opencontext.db` | — built in (`node:sqlite`) | +| **Cloudflare D1** | `d1://ACCOUNT_ID/DATABASE_ID?apiToken=TOKEN` | — built in (HTTP) | +| **DuckDB** | `duckdb:///path/to/opencontext.duckdb` | `npm i @duckdb/node-api` | +| **libSQL / Turso** | `libsql://DB.turso.io?authToken=TOKEN` | `npm i @libsql/client` | +| **PostgreSQL** | `postgres://user:pass@host:5432/db` | `npm i pg` | +| **Google Cloud SQL** | `cloudsql://user:pass@PROJECT:REGION:INSTANCE/db` | `npm i @google-cloud/cloud-sql-connector pg` | +| **MySQL / MariaDB** | `mysql://user:pass@host:3306/db` | `npm i mysql2` | +| **SQL Server / Azure SQL** | `mssql://user:pass@host:1433/db` | `npm i mssql` | +| **MongoDB** | `mongodb://user:pass@host:27017/db` | `npm i mongodb` | +| **Redis / Valkey** | `redis://host:6379` | `npm i redis` | +| **Google Firestore** | `firestore://PROJECT_ID` | `npm i @google-cloud/firestore` | +| **Amazon DynamoDB** | `dynamodb://REGION/TABLE` | `npm i @aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb` | +| **SurrealDB** | `surrealdb://user:pass@host:8000/ns/db` | `npm i surrealdb` | Drivers are **optional peer dependencies** — nothing is installed until you ask for a backend that needs it, so the default install and the Docker image stay small. Pick a backend whose @@ -352,6 +358,21 @@ deployment strategies... npm start -- convert [options] ``` +### Database commands + +```bash +opencontext db status # which backend is active, and where that came from +opencontext db adapters # every backend, and whether its driver is installed +opencontext db test "" # try a connection without saving it +opencontext db use "" # switch to it +opencontext db migrate --to "" # copy contexts and bubbles across +opencontext db reset # go back to the default JSON file +``` + +`db migrate` only ever reads the source, so it cannot damage the store you already have. Add +`--replace` to empty the target first, or `--from ` to copy between two backends without +switching to either. + ### Options | Option | Description | Default | @@ -485,8 +506,17 @@ npm test # Run tests with coverage npm run test:coverage + +# Run the store conformance suite against real databases +docker compose -f docker-compose.test.yml up -d +npm run test:backends +docker compose -f docker-compose.test.yml down -v ``` +Every backend must pass the same conformance suite (`tests/store/conformance.ts`) unmodified — +it is the only definition of correct storage behaviour. It runs against JSON, SQLite and +in-memory with no external services, covering all three shared implementations. + ### Running the full stack locally The UI talks to the backend server for all data — start both: @@ -519,10 +549,23 @@ opencontext/ │ │ └── ollama-preferences.ts # AI-powered analysis (Ollama) │ ├── utils/ │ │ └── file.ts # File I/O utilities +│ ├── store/ # BYODB — the pluggable context store +│ │ ├── index.ts # Adapter registry + factory +│ │ ├── types.ts # ContextStoreAdapter interface +│ │ ├── dsn.ts # Connection string parsing + redaction +│ │ ├── config.ts # Resolution order and saved settings +│ │ ├── manager.ts # Live connection, reconnect, swap +│ │ ├── migrate.ts # Copy one store into another +│ │ ├── adapters/ # One CRUD implementation per family +│ │ │ ├── json.ts # file +│ │ │ ├── sql.ts # all 8 SQL engines, via a Dialect +│ │ │ ├── document.ts # document/KV stores, via a DocumentDriver +│ │ │ └── surreal.ts # SurrealDB (multi-model) +│ │ └── drivers/ # Per-engine connection code │ └── mcp/ # MCP server │ ├── index.ts # Entry point (stdio transport) │ ├── server.ts # Tool definitions -│ ├── store.ts # JSON-based context store +│ ├── store.ts # Deprecated re-export of ../store │ └── types.ts # Type definitions │ └── ui/ # Web dashboard (React + Vite) @@ -532,7 +575,8 @@ opencontext/ │ ├── PreferencesEditor.tsx │ ├── ContextViewer.tsx │ ├── ConversionPipeline.tsx - │ └── VendorExport.tsx + │ ├── VendorExport.tsx + │ └── DatabaseSettings.tsx # Pick, test, and migrate backends ├── store/context.tsx # React Context state ├── types/preferences.ts # Shared types └── exporters/ # Claude, ChatGPT, Gemini exporters @@ -549,6 +593,7 @@ opencontext/ - **Ollama** - Local LLM inference (optional) - **adm-zip** - ZIP file handling - **chalk** - Terminal colors +- **Database drivers** - optional peer dependencies (`pg`, `mysql2`, `mssql`, `mongodb`, `redis`, `surrealdb`, …); SQLite uses the built-in `node:sqlite` **Web UI** - **React 19 + Vite 7** - UI framework and build tool @@ -574,7 +619,7 @@ The **open-context MCP server** lets Claude remember things across conversations | `update_context` | Update a context by ID | | `delete_context` | Delete a context by ID | -Context is stored at `~/.opencontext/contexts.json`. Set `OPENCONTEXT_STORE_PATH` to use a custom location. +Context is stored at `~/.opencontext/contexts.json` by default. Set `OPENCONTEXT_DB_URL` to keep it in [any of the 15 supported databases](#byodb) instead — the tools behave identically either way. `OPENCONTEXT_STORE_PATH` still works and simply points the JSON store somewhere else. ### Connect to Claude Code @@ -646,7 +691,8 @@ All data is stored in the mounted volume — no browser localStorage is used. Th | `preferences.json` | Your structured preferences (used by the UI form) | | `preferences.md` | Claude preferences doc — paste into Claude Settings → Preferences | | `memory.md` | Claude memory doc — paste into Claude → Manage Memory | -| `contexts.json` | MCP context entries saved by Claude | +| `contexts.json` | MCP context entries saved by Claude — the default store, unused once you configure another database | +| `config.json` | Saved database connection string, written with owner-only (`0600`) permissions | ### Environment variables @@ -655,7 +701,8 @@ All data is stored in the mounted volume — no browser localStorage is used. Th | `PORT` | `3000` | HTTP server port | | `OLLAMA_HOST` | `http://host.docker.internal:11434` | Ollama endpoint — automatically reaches Ollama running on your host machine | | `OLLAMA_MODEL` | `gpt-oss:20b` | Default model for preference analysis | -| `OPENCONTEXT_STORE_PATH` | `/root/.opencontext/contexts.json` | MCP context store path (preferences files live in the same directory) | +| `OPENCONTEXT_DB_URL` | — | Database for the context store — any [supported backend](#byodb). Takes precedence over anything saved locally | +| `OPENCONTEXT_STORE_PATH` | `/root/.opencontext/contexts.json` | Legacy JSON store path (preferences files live in the same directory). Ignored when `OPENCONTEXT_DB_URL` is set | `host.docker.internal` is a special DNS name that resolves to your host machine's IP from inside a Docker container. On Linux you may need `--add-host=host.docker.internal:host-gateway`. @@ -676,6 +723,12 @@ The server exposes a REST API alongside the UI: | `GET /api/contexts/:id` | Get a context by ID | | `PUT /api/contexts/:id` | Update a context | | `DELETE /api/contexts/:id` | Delete a context | +| `GET /api/db/status` | Active backend, where it was configured, and what it holds | +| `GET /api/db/adapters` | Every supported backend and whether its driver is installed | +| `POST /api/db/test` | Test a connection string without saving it | +| `PUT /api/db/config` | Save a connection string and switch to it | +| `DELETE /api/db/config` | Clear it and fall back to the default | +| `POST /api/db/migrate` | Copy contexts and bubbles into another backend | ### MCP stdio mode @@ -895,6 +948,7 @@ npm start -- convert export.zip --skip-preferences - **Manual Claude import** - No direct API (paste manually) - **Image references** - Images copied but not embedded - **Token limits** - Very large exports may be truncated +- **Search on document backends** - MongoDB, Redis, Firestore and DynamoDB filter in memory rather than in the database ([details](#byodb)) --- @@ -913,6 +967,7 @@ npm start -- convert export.zip --skip-preferences - [x] Export to Claude, ChatGPT, and Gemini formats - [x] Automated tests - [x] Docker support (Web UI + MCP server) +- [x] Bring your own database — 15 pluggable backends ### Future Possibilities diff --git a/ui/public/db-logos/README.md b/ui/public/db-logos/README.md new file mode 100644 index 0000000..86f3019 --- /dev/null +++ b/ui/public/db-logos/README.md @@ -0,0 +1,45 @@ +# Database backend logos + +Vendor marks identifying each supported backend in the project README and on the +**Database** settings page. The UI references them as `/db-logos/.svg`, where +`` is the DSN scheme that selects the backend. + +| Source | Files | License | +|---|---|---| +| [Simple Icons](https://github.com/simple-icons/simple-icons) | all except the two below | CC0 1.0 Universal | +| [Devicon](https://github.com/devicons/devicon) | `mssql.svg`, `dynamodb.svg` | MIT | + +Simple Icons no longer ships the Microsoft and Amazon marks, which is why those two come +from Devicon instead. + +Colours are the vendors' own, with three changed for legibility: `json.svg` (pure black, +invisible in dark mode) and `duckdb.svg` / `libsql.svg` (too light to read on white). + +Each mark is the trademark of its respective owner and is used only to identify that +vendor's product. No affiliation or endorsement is implied. + +## Devicon licence + +``` +MIT License + +Copyright (c) 2015 konpa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` diff --git a/ui/public/db-logos/cloudsql.svg b/ui/public/db-logos/cloudsql.svg new file mode 100644 index 0000000..9c06957 --- /dev/null +++ b/ui/public/db-logos/cloudsql.svg @@ -0,0 +1 @@ +Google Cloud \ No newline at end of file diff --git a/ui/public/db-logos/d1.svg b/ui/public/db-logos/d1.svg new file mode 100644 index 0000000..66cc020 --- /dev/null +++ b/ui/public/db-logos/d1.svg @@ -0,0 +1 @@ +Cloudflare \ No newline at end of file diff --git a/ui/public/db-logos/duckdb.svg b/ui/public/db-logos/duckdb.svg new file mode 100644 index 0000000..9b6170d --- /dev/null +++ b/ui/public/db-logos/duckdb.svg @@ -0,0 +1 @@ +DuckDB \ No newline at end of file diff --git a/ui/public/db-logos/dynamodb.svg b/ui/public/db-logos/dynamodb.svg new file mode 100644 index 0000000..0ada204 --- /dev/null +++ b/ui/public/db-logos/dynamodb.svg @@ -0,0 +1 @@ + diff --git a/ui/public/db-logos/firestore.svg b/ui/public/db-logos/firestore.svg new file mode 100644 index 0000000..f5ff80e --- /dev/null +++ b/ui/public/db-logos/firestore.svg @@ -0,0 +1 @@ +Firebase \ No newline at end of file diff --git a/ui/public/db-logos/json.svg b/ui/public/db-logos/json.svg new file mode 100644 index 0000000..4c45043 --- /dev/null +++ b/ui/public/db-logos/json.svg @@ -0,0 +1 @@ +JSON \ No newline at end of file diff --git a/ui/public/db-logos/libsql.svg b/ui/public/db-logos/libsql.svg new file mode 100644 index 0000000..72d2ebf --- /dev/null +++ b/ui/public/db-logos/libsql.svg @@ -0,0 +1 @@ +Turso \ No newline at end of file diff --git a/ui/public/db-logos/mongodb.svg b/ui/public/db-logos/mongodb.svg new file mode 100644 index 0000000..13d7d00 --- /dev/null +++ b/ui/public/db-logos/mongodb.svg @@ -0,0 +1 @@ +MongoDB \ No newline at end of file diff --git a/ui/public/db-logos/mssql.svg b/ui/public/db-logos/mssql.svg new file mode 100644 index 0000000..57a6a3f --- /dev/null +++ b/ui/public/db-logos/mssql.svg @@ -0,0 +1 @@ + diff --git a/ui/public/db-logos/mysql.svg b/ui/public/db-logos/mysql.svg new file mode 100644 index 0000000..0948dc4 --- /dev/null +++ b/ui/public/db-logos/mysql.svg @@ -0,0 +1 @@ +MySQL \ No newline at end of file diff --git a/ui/public/db-logos/postgres.svg b/ui/public/db-logos/postgres.svg new file mode 100644 index 0000000..931bdae --- /dev/null +++ b/ui/public/db-logos/postgres.svg @@ -0,0 +1 @@ +PostgreSQL \ No newline at end of file diff --git a/ui/public/db-logos/redis.svg b/ui/public/db-logos/redis.svg new file mode 100644 index 0000000..61480c9 --- /dev/null +++ b/ui/public/db-logos/redis.svg @@ -0,0 +1 @@ +Redis \ No newline at end of file diff --git a/ui/public/db-logos/sqlite.svg b/ui/public/db-logos/sqlite.svg new file mode 100644 index 0000000..3bfccb8 --- /dev/null +++ b/ui/public/db-logos/sqlite.svg @@ -0,0 +1 @@ +SQLite \ No newline at end of file diff --git a/ui/public/db-logos/surrealdb.svg b/ui/public/db-logos/surrealdb.svg new file mode 100644 index 0000000..6a17d53 --- /dev/null +++ b/ui/public/db-logos/surrealdb.svg @@ -0,0 +1 @@ +SurrealDB \ No newline at end of file diff --git a/ui/src/components/DatabaseSettings.tsx b/ui/src/components/DatabaseSettings.tsx index ce35951..166171e 100644 --- a/ui/src/components/DatabaseSettings.tsx +++ b/ui/src/components/DatabaseSettings.tsx @@ -60,6 +60,47 @@ const FAMILY_LABELS: Record = { document: 'Document & key-value', }; +/** Backends we ship a vendor mark for, served from `public/db-logos`. */ +const LOGO_SCHEMES = new Set([ + 'json', + 'sqlite', + 'd1', + 'duckdb', + 'libsql', + 'postgres', + 'cloudsql', + 'mysql', + 'mssql', + 'mongodb', + 'redis', + 'firestore', + 'dynamodb', + 'surrealdb', +]); + +/** + * A backend's vendor mark, on a light chip. + * + * The marks keep their own brand colours, and several of those — SQLite's navy, + * JSON's grey — vanish against this theme's black. Giving every logo the same + * light square to sit on keeps them all legible and reads as one set. Backends + * with no vendor behind them (in-memory) render nothing. + */ +function AdapterLogo({ scheme, size = 18 }: { scheme: string; size?: number }) { + if (!LOGO_SCHEMES.has(scheme)) { + return null; + } + const inner = Math.round(size * 0.72); + return ( + + + + ); +} + type Feedback = { kind: 'ok' | 'error'; message: string } | null; export default function DatabaseSettings() { @@ -192,6 +233,7 @@ export default function DatabaseSettings() { ) : ( <>
+ {status.adapter && } {status.adapter?.remote ? ( ) : ( @@ -255,12 +297,13 @@ export default function DatabaseSettings() { key={adapter.scheme} type="button" onClick={() => setUrl(adapter.example)} - className={`px-2.5 py-1 rounded-md text-xs border transition-colors ${ + className={`inline-flex items-center gap-1.5 px-2.5 py-1 rounded-md text-xs border transition-colors ${ selected?.scheme === adapter.scheme ? 'border-foreground/40 bg-accent text-foreground' : 'border-border text-muted-foreground hover:bg-accent hover:text-foreground' }`} > + {adapter.label} {!adapter.installed && ( · diff --git a/ui/src/components/__tests__/DatabaseSettings.test.tsx b/ui/src/components/__tests__/DatabaseSettings.test.tsx index 03a04b8..aaba91d 100644 --- a/ui/src/components/__tests__/DatabaseSettings.test.tsx +++ b/ui/src/components/__tests__/DatabaseSettings.test.tsx @@ -26,6 +26,15 @@ const adapters = [ family: 'sql', installed: true, }, + { + scheme: 'memory', + label: 'In-memory', + example: 'memory://', + packageName: null, + remote: false, + family: 'document', + installed: true, + }, { scheme: 'firestore', label: 'Google Firestore', @@ -196,6 +205,25 @@ describe('DatabaseSettings', () => { expect(screen.getByRole('button', { name: /save & switch/i })).toBeDisabled(); }); + it('shows the vendor mark for each backend that has one', async () => { + global.fetch = mockFetch() as unknown as typeof fetch; + const { container } = render(); + + await screen.findByText('PostgreSQL'); + expect(container.querySelector('img[src="/db-logos/postgres.svg"]')).toBeInTheDocument(); + expect(container.querySelector('img[src="/db-logos/firestore.svg"]')).toBeInTheDocument(); + // The status card names the current backend, so its mark appears there too. + expect(container.querySelectorAll('img[src="/db-logos/json.svg"]').length).toBe(2); + }); + + it('renders no mark for a backend with no vendor behind it', async () => { + global.fetch = mockFetch() as unknown as typeof fetch; + const { container } = render(); + + expect(await screen.findByText('In-memory')).toBeInTheDocument(); + expect(container.querySelector('img[src="/db-logos/memory.svg"]')).toBeNull(); + }); + it('keeps the action buttons disabled until a connection string is entered', async () => { global.fetch = mockFetch() as unknown as typeof fetch; render(); From e1909288bded8886b9dcd73eb7ba123c5ffc9dd2 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 17 Aug 2026 21:24:58 -0500 Subject: [PATCH 6/8] fix(test): create the SQL Server database the conformance suite connects to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Postgres and MySQL create their database from an environment variable. The SQL Server image has no equivalent, so `docker compose up` followed by `npm run test:backends` — the workflow the README documents — failed all 50 mssql tests. The server reports a missing default database as "Login failed for user 'sa'", which sends you looking at the password instead. Adds a healthcheck and a one-shot init service that creates the database, and records every service's connection string in the compose header, since nothing until now said what to export. Also documents that SQL Server cannot be addressed by IP while encrypting: TLS forbids an IP as the SNI server name, so a hostname (or `?encrypt=false` on a trusted network) is required. Verified from a clean slate: `down -v`, `up -d`, 450 conformance tests passing across 9 live backends. Claude-Session: https://claude.ai/code/session_01CrC2FRcZVEti9kGyhsxaip --- README.md | 9 +++++++++ docker-compose.test.yml | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/README.md b/README.md index cb20c6a..30543cb 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,11 @@ Drivers are **optional peer dependencies** — nothing is installed until you as that needs it, so the default install and the Docker image stay small. Pick a backend whose driver is missing and opencontext tells you exactly what to run. +> **SQL Server and IP addresses.** The driver encrypts by default, and TLS forbids an IP +> address as the SNI server name, so `mssql://…@10.0.0.5:1433/db` fails with a `servername` +> error. Address the server by hostname, or add `?encrypt=false` for a local instance on a +> trusted network. + ### Managed services Most managed databases speak a protocol already listed above, so they need no special support: @@ -513,6 +518,10 @@ npm run test:backends docker compose -f docker-compose.test.yml down -v ``` +`docker-compose.test.yml` lists the connection string to export for each service. A backend +whose connection string is not in the environment is skipped, so the suite is useful with any +subset of them running. + Every backend must pass the same conformance suite (`tests/store/conformance.ts`) unmodified — it is the only definition of correct storage behaviour. It runs against JSON, SQLite and in-memory with no external services, covering all three shared implementations. diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 6a0ea5b..06081c0 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -6,6 +6,20 @@ # # Every service binds a non-default host port so it cannot collide with a real # database already running on the machine. +# +# The connection strings the suite expects: +# +# OPENCONTEXT_TEST_POSTGRES_URL="postgres://opencontext:opencontext@127.0.0.1:55432/opencontext" +# OPENCONTEXT_TEST_MYSQL_URL="mysql://opencontext:opencontext@127.0.0.1:53306/opencontext" +# OPENCONTEXT_TEST_MSSQL_URL="mssql://sa:OpenContext!2026@localhost:51433/opencontext" +# OPENCONTEXT_TEST_MONGODB_URL="mongodb://127.0.0.1:57017/opencontext" +# OPENCONTEXT_TEST_REDIS_URL="redis://127.0.0.1:56379" +# OPENCONTEXT_TEST_SURREALDB_URL="surrealdb://root:root@127.0.0.1:58000/test/test" +# OPENCONTEXT_TEST_DYNAMODB_URL="dynamodb://us-east-1/opencontext?endpoint=http://127.0.0.1:58001&accessKeyId=test&secretAccessKey=test" +# OPENCONTEXT_TEST_DUCKDB=1 +# +# SQL Server must be addressed as `localhost` rather than `127.0.0.1`: the driver +# encrypts by default, and TLS forbids an IP address as the SNI server name. services: postgres: @@ -43,6 +57,29 @@ services: MSSQL_PID: Developer ports: - '51433:1433' + healthcheck: + test: + - CMD-SHELL + - /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "$$MSSQL_SA_PASSWORD" -C -Q "SELECT 1" + interval: 5s + retries: 30 + start_period: 20s + + # Postgres and MySQL create their database from an environment variable; the + # SQL Server image has no equivalent, so without this the suite connects to a + # database that does not exist and the server reports it as a login failure. + mssql-init: + image: mcr.microsoft.com/mssql/server:2022-latest + depends_on: + mssql: + condition: service_healthy + restart: on-failure + entrypoint: + - /bin/bash + - -c + - >- + /opt/mssql-tools18/bin/sqlcmd -S mssql -U sa -P 'OpenContext!2026' -C + -Q "IF DB_ID('opencontext') IS NULL CREATE DATABASE opencontext" mongodb: image: mongo:8 From 8477c7c65ecdee655bb3f6983fa1d80dc26ab84a Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 17 Aug 2026 21:27:03 -0500 Subject: [PATCH 7/8] fix(deps): declare the driver versions the backends were verified against Three optional peer ranges excluded the very versions UAT ran against, so installing the driver the README recommends produced a peer warning: mongodb ^6.0.0 but 7.5.0 was tested mssql ^11.0.0 but 12.7.0 was tested redis ^5.0.0 but 6.2.1 was tested Each range now covers both majors. `@duckdb/node-api` still reads as a mismatch because upstream publishes its releases as `1.5.5-r.4`-style prereleases, which no ordinary range matches without `includePrerelease`; `^1.0.0` remains the right declaration for that line. Claude-Session: https://claude.ai/code/session_01CrC2FRcZVEti9kGyhsxaip --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index bcf174a..0f6007b 100644 --- a/package.json +++ b/package.json @@ -70,11 +70,11 @@ "@google-cloud/cloud-sql-connector": "^1.0.0", "@google-cloud/firestore": "^7.0.0", "@libsql/client": "^0.15.0", - "mongodb": "^6.0.0", - "mssql": "^11.0.0", + "mongodb": "^6.0.0 || ^7.0.0", + "mssql": "^11.0.0 || ^12.0.0", "mysql2": "^3.0.0", "pg": "^8.0.0", - "redis": "^5.0.0", + "redis": "^5.0.0 || ^6.0.0", "surrealdb": "^2.0.0" }, "peerDependenciesMeta": { From c4c0846a783b3f855d82192bfa47273c71d4e6a3 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 17 Aug 2026 21:39:11 -0500 Subject: [PATCH 8/8] feat(ui): show the supported databases as a marquee on the landing page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A "Supports all of these" section between Features and How it works, scrolling the vendor mark for all 14 backends that have one. In-memory is absent deliberately: there is no vendor behind it and so no logo to show. The track holds two identical copies of the list and shifts by exactly half its width, which only loops seamlessly if the trailing gap is padded to match the gaps between items — otherwise half the track is half a gap short and the seam jumps. Verified in a real browser: 28 marks, zero drift at the seam. A marquee is unreadable to a screen reader, so the chips are aria-hidden and the container carries the full list as an aria-label. Animation is dropped entirely under prefers-reduced-motion, and pauses on hover. Also fixes the two stale UI tests this branch had been carrying. Neither was a code bug: the hero assertion still expected copy rewritten in 0f24191, and the active-nav test rendered Layout at "/" — the public landing page, which renders outside Layout, so nothing in the sidebar is ever active there. That made one test fail and its sibling pass for the wrong reason. The UI suite is now fully green at 274 tests. Claude-Session: https://claude.ai/code/session_01CrC2FRcZVEti9kGyhsxaip --- ui/src/components/Landing.tsx | 69 ++++++++++++++++++++ ui/src/components/__tests__/Landing.test.tsx | 22 ++++++- ui/src/components/__tests__/Layout.test.tsx | 8 +-- ui/src/index.css | 27 ++++++++ 4 files changed, 121 insertions(+), 5 deletions(-) diff --git a/ui/src/components/Landing.tsx b/ui/src/components/Landing.tsx index 182f684..f43151a 100644 --- a/ui/src/components/Landing.tsx +++ b/ui/src/components/Landing.tsx @@ -58,6 +58,30 @@ const HOW_IT_WORKS = [ }, ]; +/** + * Every backend the context store supports, in the order they scroll past. + * + * `scheme` is both the DSN scheme it is selected with and the filename of its + * vendor mark in `public/db-logos`. The in-memory backend is deliberately absent + * — there is no vendor behind it and so no logo to show. + */ +const BACKENDS = [ + { scheme: 'postgres', label: 'PostgreSQL' }, + { scheme: 'sqlite', label: 'SQLite' }, + { scheme: 'mongodb', label: 'MongoDB' }, + { scheme: 'mysql', label: 'MySQL' }, + { scheme: 'redis', label: 'Redis' }, + { scheme: 'duckdb', label: 'DuckDB' }, + { scheme: 'surrealdb', label: 'SurrealDB' }, + { scheme: 'd1', label: 'Cloudflare D1' }, + { scheme: 'mssql', label: 'SQL Server' }, + { scheme: 'dynamodb', label: 'DynamoDB' }, + { scheme: 'firestore', label: 'Firestore' }, + { scheme: 'libsql', label: 'libSQL / Turso' }, + { scheme: 'cloudsql', label: 'Cloud SQL' }, + { scheme: 'json', label: 'JSON file' }, +]; + const DOCKER_CMD = 'docker run -p 3000:3000 -v opencontext-data:/root/.opencontext adityakarnam/opencontext:latest'; const MCP_SNIPPET = `{ @@ -110,6 +134,7 @@ export default function Landing() {
+ {/* ------------------------------------------------------------------ */} + {/* Databases */} + {/* ------------------------------------------------------------------ */} +
+
+

+ Bring your own database +

+

+ Supports all of these +

+

+ Start on the zero-config JSON file. Outgrow it and point open-context at whatever you + already run — the CLI, the web UI, and the MCP tools all behave exactly the same. +

+
+ + {/* Two identical copies scroll as one track; see `animate-marquee` in index.css. */} +
b.label).join(', ')}`} + > +
+ {[...BACKENDS, ...BACKENDS].map(({ scheme, label }, index) => ( + + ))} +
+
+ +

+ Fifteen backends in all, and your data never leaves infrastructure you control. +

+
+ {/* ------------------------------------------------------------------ */} {/* How it works */} {/* ------------------------------------------------------------------ */} diff --git a/ui/src/components/__tests__/Landing.test.tsx b/ui/src/components/__tests__/Landing.test.tsx index 6ce919e..298f03c 100644 --- a/ui/src/components/__tests__/Landing.test.tsx +++ b/ui/src/components/__tests__/Landing.test.tsx @@ -76,10 +76,30 @@ describe('Landing', () => { renderWithProviders(); expect( - screen.getByText(/open-context migrates your full conversation history/i) + screen.getByText(/import chat history from any AI platform/i) ).toBeInTheDocument(); }); + it('should render the supported databases section', () => { + const { container } = renderWithProviders(); + + expect(screen.getByText('Supports all of these')).toBeInTheDocument(); + expect(screen.getByText('Bring your own database')).toBeInTheDocument(); + // A vendor mark for each backend, twice over: the marquee scrolls two + // identical copies so the loop has no visible seam. + expect(container.querySelectorAll('img[src^="/db-logos/"]').length).toBe(28); + expect(container.querySelectorAll('img[src="/db-logos/postgres.svg"]').length).toBe(2); + }); + + it('names every supported database for screen readers, which cannot read the marquee', () => { + const { container } = renderWithProviders(); + + const marquee = container.querySelector('[aria-label^="Supported databases"]'); + expect(marquee).toBeInTheDocument(); + expect(marquee?.getAttribute('aria-label')).toContain('PostgreSQL'); + expect(marquee?.getAttribute('aria-label')).toContain('SurrealDB'); + }); + it('should render how it works section', () => { renderWithProviders(); diff --git a/ui/src/components/__tests__/Layout.test.tsx b/ui/src/components/__tests__/Layout.test.tsx index 2f0caa2..93aa635 100644 --- a/ui/src/components/__tests__/Layout.test.tsx +++ b/ui/src/components/__tests__/Layout.test.tsx @@ -62,10 +62,10 @@ describe('Layout', () => { renderWithProviders( }> - Home} /> + Home} /> , - { initialEntries: ['/'] } + { initialEntries: ['/dashboard'] } ); const dashboardLink = screen.getByText('Dashboard').closest('a'); @@ -76,10 +76,10 @@ describe('Layout', () => { renderWithProviders( }> - Home} /> + Home} /> , - { initialEntries: ['/'] } + { initialEntries: ['/dashboard'] } ); const preferencesLink = screen.getByText('Preferences').closest('a'); diff --git a/ui/src/index.css b/ui/src/index.css index a76c264..b28e76d 100644 --- a/ui/src/index.css +++ b/ui/src/index.css @@ -160,3 +160,30 @@ min-height: 100vh; } } + +/* ===== Database marquee ===== */ +/* + * The track holds two identical copies of the backend list, so shifting it by + * exactly half its width lands the second copy where the first began and the + * loop is seamless. That only holds if the trailing gap is padded to match the + * gaps between items — otherwise half the track is half a gap short and the + * seam visibly jumps. + */ +@theme { + --animate-marquee: marquee 45s linear infinite; +} + +@keyframes marquee { + from { + transform: translateX(0); + } + to { + transform: translateX(-50%); + } +} + +@media (prefers-reduced-motion: reduce) { + .animate-marquee { + animation: none; + } +}