From 4835b21a5348c455c36aed43b1c85ed3b59ea0cb Mon Sep 17 00:00:00 2001 From: Nabil Mouzouna Date: Wed, 25 Mar 2026 16:39:20 +0100 Subject: [PATCH] DOCS: writing ADR-005-file-storage-strategy.md for the last SERVICE in our BaaS --- apps/api/data/appbase.sqlite-shm | Bin 32768 -> 32768 bytes apps/api/data/appbase.sqlite-wal | Bin 1133032 -> 1268992 bytes apps/api/src/plugins/infrastructure.ts | 5 + apps/api/src/routes/db.ts | 210 +++++++++++++++++++++- apps/dashboard/next-env.d.ts | 6 + docs/API-SPEC.md | 1 + docs/ARCHITECTURE.md | 1 + docs/adr/ADR-005-file-storage-strategy.md | 160 +++++++++++++++++ 8 files changed, 379 insertions(+), 4 deletions(-) create mode 100644 apps/dashboard/next-env.d.ts create mode 100644 docs/adr/ADR-005-file-storage-strategy.md diff --git a/apps/api/data/appbase.sqlite-shm b/apps/api/data/appbase.sqlite-shm index f13982b605b51562b70d580fbb3814f1d2969993..acadd027b56d830e12e8581d34cce79e34065f4f 100644 GIT binary patch delta 368 zcmZo@U}|V!s+V}A%K!t63=9G$KtdWQP+gzVdR+H$f!U001FSJ=;XaD;*(FX@WAlq6D%Dzo3A)JF!L%f zsxYcEYBB0E8Za797IeD3`Hw3P8>7T#!??qYo6q=KF)~Vm*`jg`9E<{t!i7AFH1-O@6?_Hu*)e5Th7^`z%d}Q5??Q{3Mf4h+BzKjZu?Phf$x=2q<%tY4Wu? LAx34Oz2b}j_X2Y{ delta 285 zcmZo@U}|V!s+V}A%K!t63=9IoKtdWQAgLkgz%IYMN<=2_?e2`=IY;hS0{6lxyihwIcZ7eCkG|1Nm~+1+LLe+Nuo(C>3G(8 zvMSZ}GshrjQ9*I=(~Xknhrthva`(o(mHL#|9j-z4j;;>>D|nrM&*Ss-C#%sT*70@| z_+(aVJ74N7#wI+9fS=ir0u#0qQ_zplP3UaoE95uGQlwz#r_KL_=T+HYFH&hWnaxhq z5Nlmk3m;rrtx!}O=~>=d-x#lNrdy(6rgc$$Rb4zD6eWLPil{XCgX82-6ELnB&#F6O z5jCtgKIOo*rU0kLt^gtu)V@ldAg& zb?KuEP-{-Huj25J9xA)Fyk}Fa!h{{bp2K|DW^58x@!Zq)LabnLUS-ZD$FrqcO3}A& zJh{c+{N~9i>kU^}^N*O&caY!08|?vTw#_~0OujDuA$NCvVvXC1dA;6|U=jGNsWqGH zR(czJH4$=|$kZ;-*fx@{ZKY+pIzGKMPDg#!eO;oj-^*cDQG-#x7~ohX!4d+;5FE=g zgrvy~r+6rVA$eLRc#@hxGYYFPD)^okM6jG>IE|ushEyny<5iIsG>TPdUS?UEBRF0p zBuS%and3-8P-KClJeo{#iYmwqBe9x5(;iY`1ePR8k4Mnhau4re1d0$fS>QEBRArhZ zRF>5gjie|+%7j|?;^<)O@JEmS@h^K%1?@&WtZ1hRyB9r+&B1iEQ-9ZG-7%$f&Lv6A3m|QlH62lL~2O>#MFTpIwmJy-#QtB^AoW`8;Fk?LCoe^VpNzO zk!#7iw#cm5EPpFK(;o{hSy;cEXk5uRM*H&o&HSv{;m{1KwI+PaoP}Y@M>Mvz%;h}N zVqep|72Xy8j;<4#3&)0}&8EIO;7~5jy5rVj>(Zy*2qqT4Wq4u;eA~=An2wde zr}gEXjt}9q0n)(V|?i=kQJF+vz<= z_H~a!{imlKCvsCAEoKD!?AEDId@z3PSZ0SYTH+c2-Lu{LEs=(z2h;U)H*Srb}!`#V23qEHAQx+uKpJ;*d5 z`=iKVtO?lxAHwcMM(AC;?N6q|Gq*244mCo4&APaK@pSd^caPuKd!im)nBmj#AwwD4 z(xzwi6@nhqS;^<~+~U)%$BrKSbD-U@I1<`rLgrvU!0Z@h&ToU5;kavR=STm8Um2(rOsv7gDoG_(WmM82?}0f+ClOYQX*t7oWp zO{LjP5cQsU^wel%Sx~~~)--C>(`%GyND1|E0ASx8^+yA$dxBe36xYqZ+67Z*&vb#K zxAwRDza)I!EbG80&S0!56YNp^{QtXokda@gM-^@oDs4+Ei4qbt_z@@uZ>GBvEa zMB@p@T~b(;V*WtX6$u4m#^Q+Exb1r5(KnVkEAmp6dtt=NRGKQRhgYWbwiLD&E4=Jw zEU8bAqswm^3c`5KK})6`s(kF!EB`V?Jo+m3!gSbt0CADNV`a|kal^=PD)EQK`|Pkg2oA_6qd&siIH$#q%>UP zXi<}RnP5q=#E@(@+R9$r#*QK{()#m^v)oebKeJ@SmL*Sg85+-rw_TudcpHe+W*Dfv zB%|_dj@+yAShMo&s(kA}W?O1T1v2=%nrhI|I}UZfHf4J{)w<57S{Df58=$PTq%%$J z-%JC7{`Tt1SXam>J+1zT>hd==2h=7o^|+eEU`z}I;^(T)_I4)ED4Fe|*{nZQ<{Xa_!C?zi0=yO7tvHotHdA7%i^ zYQO6e0NL*X;#pw_2pW|^P&OXuF9huvNEBuebe)44Pkbi_mZr6O&9gJMG+efe+HVH~ z%l~d0rC&h8bQ@K(>X}{PiAC+dFhE#l>%IaAZQU0RkF1YR{$5>FFl1`4ww# zpU@Y8pvNFU(6S6cvvFL15mX;Yl4rPa9pf2KTyJNu)Xpy0@+aiu&IjN9|7&OWT>$ak zcJ{Wf#&0(L1FN~)pm@1$&wog9_%Y-G^m*h)c$ICB9nv?9b_9Ewu5lzMkTi|+EDzdS zMV4_v<~%q@f*1Of$jOw(l^aaYhGEwZ|HJ@}?8d}CB8z`ql)m4TSoMhZ@Fs)!5Tss) p_>c;iUu7`AJj48K9NS;a-#3u8CqugHg(u^Q>+?g#6W_7<;osq-a!mjL delta 43 ycmZqJ<@;i?YeNfT3sVbo3rh=Y3tJ0&3r7oQ3s(zw3r`Dg3ttO=i@+Aa7oGr8N)Cnq diff --git a/apps/api/src/plugins/infrastructure.ts b/apps/api/src/plugins/infrastructure.ts index cc70cfe..8d8593a 100644 --- a/apps/api/src/plugins/infrastructure.ts +++ b/apps/api/src/plugins/infrastructure.ts @@ -32,6 +32,11 @@ export async function registerInfrastructure(app: FastifyInstance, env: AppEnv) version: "0.1.0", }, servers: [{ url: env.BASE_URL }], + tags: [ + { name: "auth", description: "Registration, login, session, and tokens." }, + { name: "database", description: "User-scoped collections and records (Bearer JWT)." }, + { name: "system", description: "Operational and health endpoints." }, + ], components: { securitySchemes: { bearerAuth: { type: "http", scheme: "bearer" }, diff --git a/apps/api/src/routes/db.ts b/apps/api/src/routes/db.ts index 059e46f..368c4f4 100644 --- a/apps/api/src/routes/db.ts +++ b/apps/api/src/routes/db.ts @@ -106,6 +106,204 @@ function parseFilter(filterStr: string | undefined): Record | n } } +const dbApiErrorSchema = { + type: "object", + properties: { + success: { type: "boolean", const: false }, + error: { + type: "object", + properties: { code: { type: "string" }, message: { type: "string" } }, + required: ["code", "message"], + }, + }, + required: ["success", "error"], +} as const; + +const dbRecordSchema = { + type: "object", + properties: { + id: { type: "string" }, + collection: { type: "string" }, + ownerId: { type: "string" }, + data: { type: "object", additionalProperties: true }, + createdAt: { type: "string", format: "date-time" }, + updatedAt: { type: "string", format: "date-time" }, + }, + required: ["id", "collection", "ownerId", "data", "createdAt", "updatedAt"], +} as const; + +const collectionParamsSchema = { + type: "object", + required: ["collection"], + properties: { + collection: { + type: "string", + description: "Collection name (letters, digits, hyphens, underscores; max 64).", + }, + }, +} as const; + +const collectionIdParamsSchema = { + type: "object", + required: ["collection", "id"], + properties: { + collection: { + type: "string", + description: "Collection name (letters, digits, hyphens, underscores; max 64).", + }, + id: { type: "string", description: "Record id." }, + }, +} as const; + +const dbSecurityBearer = [{ bearerAuth: [] }] as const; + +const dbPostCreateOpenApi = { + tags: ["database"], + summary: "Create record", + description: "Insert a JSON document into a user-scoped collection.", + security: dbSecurityBearer, + params: collectionParamsSchema, + body: { + type: "object", + properties: { + data: { type: "object", additionalProperties: true, description: "JSON object to store (required by API)." }, + }, + }, + response: { + 201: { + type: "object", + properties: { + success: { type: "boolean", const: true }, + data: dbRecordSchema, + }, + required: ["success", "data"], + }, + 400: dbApiErrorSchema, + 401: dbApiErrorSchema, + 500: dbApiErrorSchema, + }, +} as const; + +const dbGetListOpenApi = { + tags: ["database"], + summary: "List records", + description: "Paginated list of records in a collection, with optional JSON filter on top-level data keys.", + security: dbSecurityBearer, + params: collectionParamsSchema, + querystring: { + type: "object", + properties: { + limit: { type: "string", description: "Page size (1–100, default 50)." }, + offset: { type: "string", description: "Offset for pagination." }, + filter: { type: "string", description: "URL-encoded JSON object of field equality filters." }, + }, + }, + response: { + 200: { + type: "object", + properties: { + success: { type: "boolean", const: true }, + data: { + type: "object", + properties: { + items: { type: "array", items: dbRecordSchema }, + total: { type: "integer" }, + }, + required: ["items", "total"], + }, + }, + required: ["success", "data"], + }, + 400: dbApiErrorSchema, + 401: dbApiErrorSchema, + }, +} as const; + +const dbSubscribeOpenApi = { + tags: ["database"], + summary: "Subscribe to collection changes", + description: "Server-Sent Events stream of created, updated, and deleted records for this collection.", + security: dbSecurityBearer, + params: collectionParamsSchema, + response: { + 200: { + description: "Event stream (text/event-stream).", + type: "string", + }, + 400: dbApiErrorSchema, + 401: dbApiErrorSchema, + }, +} as const; + +const dbGetOneOpenApi = { + tags: ["database"], + summary: "Get record", + description: "Fetch a single record by collection and id.", + security: dbSecurityBearer, + params: collectionIdParamsSchema, + response: { + 200: { + type: "object", + properties: { + success: { type: "boolean", const: true }, + data: dbRecordSchema, + }, + required: ["success", "data"], + }, + 400: dbApiErrorSchema, + 401: dbApiErrorSchema, + 404: dbApiErrorSchema, + }, +} as const; + +const dbPutUpdateOpenApi = { + tags: ["database"], + summary: "Update record", + description: "Replace stored JSON data for a record.", + security: dbSecurityBearer, + params: collectionIdParamsSchema, + body: { + type: "object", + properties: { + data: { type: "object", additionalProperties: true, description: "JSON object to store (required by API)." }, + }, + }, + response: { + 200: { + type: "object", + properties: { + success: { type: "boolean", const: true }, + data: dbRecordSchema, + }, + required: ["success", "data"], + }, + 400: dbApiErrorSchema, + 401: dbApiErrorSchema, + 404: dbApiErrorSchema, + }, +} as const; + +const dbDeleteOpenApi = { + tags: ["database"], + summary: "Delete record", + description: "Remove a record from a collection.", + security: dbSecurityBearer, + params: collectionIdParamsSchema, + response: { + 200: { + type: "object", + properties: { + success: { type: "boolean", const: true }, + data: { type: "object", properties: { deleted: { type: "boolean", const: true } }, required: ["deleted"] }, + }, + required: ["success", "data"], + }, + 400: dbApiErrorSchema, + 401: dbApiErrorSchema, + 404: dbApiErrorSchema, + }, +} as const; + export async function registerDbRoutes(app: FastifyInstance) { await app.register( async (instance) => { @@ -117,7 +315,7 @@ export async function registerDbRoutes(app: FastifyInstance) { instance.post<{ Params: { collection: string }; Body: { data?: unknown }; - }>("/collections/:collection", async (request, reply) => { + }>("/collections/:collection", { schema: dbPostCreateOpenApi }, async (request, reply) => { const userId = requireUserId(request); if (!userId) { return reply.status(401).send(apiError("INVALID_TOKEN", "The provided access token is invalid or expired.")); @@ -182,7 +380,10 @@ export async function registerDbRoutes(app: FastifyInstance) { }); // GET /db/collections/:collection/subscribe — must be before :id - instance.get<{ Params: { collection: string } }>("/collections/:collection/subscribe", async (request, reply) => { + instance.get<{ Params: { collection: string } }>( + "/collections/:collection/subscribe", + { schema: dbSubscribeOpenApi }, + async (request, reply) => { const userId = requireUserId(request); if (!userId) { return reply.status(401).send(apiError("INVALID_TOKEN", "The provided access token is invalid or expired.")); @@ -219,7 +420,7 @@ export async function registerDbRoutes(app: FastifyInstance) { instance.get<{ Params: { collection: string }; Querystring: { limit?: string; offset?: string; filter?: string }; - }>("/collections/:collection", async (request, reply) => { + }>("/collections/:collection", { schema: dbGetListOpenApi }, async (request, reply) => { const userId = requireUserId(request); if (!userId) { return reply.status(401).send(apiError("INVALID_TOKEN", "The provided access token is invalid or expired.")); @@ -296,7 +497,7 @@ export async function registerDbRoutes(app: FastifyInstance) { instance.put<{ Params: { collection: string; id: string }; Body: { data?: unknown }; - }>("/collections/:collection/:id", async (request, reply) => { + }>("/collections/:collection/:id", { schema: dbPutUpdateOpenApi }, async (request, reply) => { const userId = requireUserId(request); if (!userId) { return reply.status(401).send(apiError("INVALID_TOKEN", "The provided access token is invalid or expired.")); @@ -363,6 +564,7 @@ export async function registerDbRoutes(app: FastifyInstance) { // DELETE /db/collections/:collection/:id instance.delete<{ Params: { collection: string; id: string } }>( "/collections/:collection/:id", + { schema: dbDeleteOpenApi }, async (request, reply) => { const userId = requireUserId(request); if (!userId) { diff --git a/apps/dashboard/next-env.d.ts b/apps/dashboard/next-env.d.ts new file mode 100644 index 0000000..c4b7818 --- /dev/null +++ b/apps/dashboard/next-env.d.ts @@ -0,0 +1,6 @@ +/// +/// +import "./.next/dev/types/routes.d.ts"; + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/docs/API-SPEC.md b/docs/API-SPEC.md index ba3a683..abb6976 100644 --- a/docs/API-SPEC.md +++ b/docs/API-SPEC.md @@ -548,5 +548,6 @@ data: {"type":"created","collection":"passwords","record":{"id":"rec_123","site" - This file defines the public AppBase contract. - `docs/ARCHITECTURE.md` explains how that contract is hosted and routed. - `better-auth` is an internal implementation choice; the public contract is cookie session + JWT access as defined here and in ADR-003. +- **Storage (`/storage/*`):** backend strategy decisions (FS-first driver abstraction, container volume persistence, metadata-based versioning) are recorded in `docs/adr/ADR-005-file-storage-strategy.md`. - **Database (`/db/*`):** implementation decisions (owner scoping, M1 filter semantics, SSE in-process bus) are recorded in `docs/adr/ADR-004-database-api-service.md`. - Dashboard authentication is intentionally separate from the public API contract. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1cbc596..09aa68b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -464,3 +464,4 @@ Architecture decisions that shaped this document: | [ADR-002 — ORM and Migration Strategy](./adr/ADR-002-orm-and-migration-strategy.md) | Drizzle ORM + `better-sqlite3` + `drizzle-kit` | Schema tables in §4 sourced directly from `packages/db/src/schema/`; `createDb(path)` factory enables per-app SQLite in M2+ | | [ADR-003 — Auth Implementation](./adr/ADR-003-auth-implementation.md) | 3-token model (refresh token + JWT + API key); argon2id hashing; EdDSA signing | Auth flow in §3.1; `refresh_tokens` and `api_keys` schema in §4; JWT-on-hot-path pattern in §3.2 | | [ADR-004 — Database API Service](./adr/ADR-004-database-api-service.md) | Fastify `/db/*` plugin; `records` document model; owner isolation; in-process SSE | §3.4 sequence; `records` in §4; `/db/*` routes in §5; complements API-SPEC §7 | +| [ADR-005 — File Storage Strategy](./adr/ADR-005-file-storage-strategy.md) | FS-first storage driver abstraction; container volume persistence; metadata-based versioning | §3.3 storage flow, `files.storage_path` in §4, persistence layout in §7; complements API-SPEC §6 | diff --git a/docs/adr/ADR-005-file-storage-strategy.md b/docs/adr/ADR-005-file-storage-strategy.md new file mode 100644 index 0000000..f283c62 --- /dev/null +++ b/docs/adr/ADR-005-file-storage-strategy.md @@ -0,0 +1,160 @@ +# ADR-005 — File Storage Strategy + +**Status:** Accepted +**Date:** 2026-03-19 +**Deciders:** AppBase core team +**Tags:** `backend`, `storage`, `filesystem`, `docker`, `architecture` + +--- + +## Context + +AppBase exposes file APIs under `/storage/*` and persists file metadata in the `files` table (`id`, `bucket`, `filename`, `mime_type`, `size`, `storage_path`, `owner_id`, `created_at`). The current architecture explicitly models file bytes on disk (`data/storage/...`) and metadata in SQLite (`appbase.sqlite`) for M1, then per-app isolation in M2+ (`data/{appId}/app.sqlite` + `data/{appId}/storage/`). + +Deployment model is also explicit: **one app = one container** for M1 BaaS units. Therefore, file persistence cannot rely on the ephemeral container writable layer; it must use mounted host volumes. + +We need a storage strategy that is: + +- Production-reasonable for LAN/offline/self-hosted VPS usage +- Simple to run with minimal dependencies in M1 +- Extensible to S3-compatible object stores later without rewriting `/storage/*` business logic +- Compatible with future versioning and retention policies + +Options considered in this ADR: + +1. Local filesystem with structured paths +2. Object storage abstraction (S3-compatible readiness) +3. SQLite BLOB storage + +We also need a versioning decision: + +- Filename suffix versioning (`report_v2.pdf`) +- Metadata-based version tracking (version history in DB) + +--- + +## Decision + +### 1. Storage backend + +Adopt a **storage driver abstraction** and ship **local filesystem** as the default/first implementation. + +- M1 runtime uses local FS for file bytes. +- File metadata remains in SQLite (`files` table and future version metadata tables if needed). +- `/storage/*` routes and service logic depend on an internal interface (e.g. `StorageDriver`) rather than directly on `fs`. + +### 2. Container persistence contract + +In containerized deployment, both database and file bytes must live on mounted volumes: + +- SQLite: `/app/data/appbase.sqlite` (M1 single app) +- File bytes: `/app/data/storage/...` + +For M2+ per-app isolation: + +- SQLite: `/app/data/{appId}/app.sqlite` +- File bytes: `/app/data/{appId}/storage/...` + +Using the container writable layer for durable data is prohibited for production. + +### 3. Object-store readiness + +The abstraction must support adding an S3-compatible adapter later (MinIO/AWS S3/R2 style APIs) without changing route contracts. + +- M1 ships FS adapter only (unless explicitly enabled by later ticket) +- Future object-store adapter is additive and selected by config (`STORAGE_DRIVER=fs|s3`) + +### 4. SQLite BLOB stance + +Reject SQLite BLOB as the primary file storage backend. + +Reasons: + +- Poor operational profile for medium/large files (DB growth, backup/restore weight, vacuum pressure) +- Blurs transactional metadata concerns with large binary I/O +- Harder to evolve toward external object storage later + +### 5. Versioning strategy + +Use **metadata tracking** as the source of truth, not filename suffix parsing. + +- Object keys/physical paths should be opaque and immutable (ID-based) +- Version semantics live in metadata (version number, logical file ID, created timestamp, pointer to storage object, etc.) +- Human filename suffixes may exist as display conventions only, never as canonical version logic + +--- + +## Option Evaluation + +### Option A — Local filesystem only (no abstraction) + +**Pros** +- Simplest implementation now +- Fast and predictable on single-node LAN/VPS + +**Cons** +- Couples business logic tightly to local FS +- Makes S3/MinIO migration more expensive later + +**Assessment:** Partially accepted; use FS now, but behind an abstraction. + +### Option B — Object storage abstraction with FS-first adapter + +**Pros** +- Best balance: simple M1 runtime + future-proof design +- Enables S3-compatible support later without API break +- Keeps storage concerns isolated behind one interface + +**Cons** +- Slightly more upfront design work + +**Assessment:** Accepted. + +### Option C — SQLite BLOB storage + +**Pros** +- Single persistence primitive +- Very small prototype simplicity + +**Cons** +- Degrades operationally as file sizes/count grow +- Slower and heavier backups +- Unnecessary coupling to DB internals + +**Assessment:** Rejected. + +--- + +## Consequences + +**Positive** + +- Aligns with current architecture and API-SPEC storage shape +- Works well for offline/LAN/personal-server use cases +- Clear migration path to S3-compatible backends +- Cleaner backup strategy (volume + DB metadata) + +**Negative / risks** + +- Requires robust filesystem hygiene (path validation, safe writes, cleanup) +- Need consistency jobs/guards for orphan files or stale metadata +- S3 adapter remains future work and must be validated separately + +--- + +## Operational Guardrails (normative for implementation tickets) + +- Validate and normalize bucket/file path components to prevent traversal. +- Write uploads atomically (temp file + rename). +- Store checksum/hash in metadata where feasible. +- Enforce max upload size and content-type allowlist policy. +- Ensure delete path removes file bytes and metadata coherently. +- Add periodic reconciliation tooling (detect metadata without object and object without metadata). + +--- + +## References + +- [`ARCHITECTURE.md`](../ARCHITECTURE.md) — §3.3 storage flow, §4 `files` schema, §7 data layout +- [`API-SPEC.md`](../API-SPEC.md) — §6 Storage Endpoints +- [ADR-001](./ADR-001-api-framework-selection.md), [ADR-002](./ADR-002-orm-and-migration-strategy.md), [ADR-003](./ADR-003-auth-implementation.md)