From 9bfcba3a302ef2e9fd53f8c9fa778f83118a56a5 Mon Sep 17 00:00:00 2001 From: Nabil Mouzouna Date: Wed, 25 Mar 2026 17:08:45 +0100 Subject: [PATCH 01/11] FEAT: implementing storage service as filesystem using FS driver for now, also considering containerizing infuture --- .github/workflows/ci.yml | 3 + apps/api/.env.example | 5 + apps/api/README.md | 11 +- apps/api/data/appbase.sqlite-shm | Bin 32768 -> 32768 bytes apps/api/data/appbase.sqlite-wal | Bin 1268992 -> 1285472 bytes apps/api/package.json | 4 +- apps/api/scripts/storage-reconcile.ts | 22 ++ apps/api/src/app.ts | 2 + apps/api/src/config/env.ts | 51 ++- apps/api/src/constants.ts | 4 +- apps/api/src/index.ts | 2 +- apps/api/src/plugins/infrastructure.ts | 3 +- apps/api/src/plugins/storage.ts | 26 ++ apps/api/src/routes/index.ts | 2 + apps/api/src/routes/storage.test.ts | 280 ++++++++++++++ apps/api/src/routes/storage.ts | 342 ++++++++++++++++++ apps/api/src/storage/factory.ts | 9 + apps/api/src/storage/reconcile.test.ts | 78 ++++ apps/api/src/storage/reconcile.ts | 34 ++ apps/api/src/types/fastify.d.ts | 2 + docs/API-SPEC.md | 2 + .../0007_files_checksum_versioning.sql | 4 + packages/db/migrations/meta/_journal.json | 7 + packages/db/src/schema/storage.ts | 6 + packages/storage/package.json | 24 ++ packages/storage/src/driver.ts | 23 ++ packages/storage/src/fs-driver.test.ts | 46 +++ packages/storage/src/fs-driver.ts | 98 +++++ packages/storage/src/index.ts | 3 + packages/storage/src/validation.test.ts | 29 ++ packages/storage/src/validation.ts | 38 ++ packages/storage/tsconfig.json | 11 + packages/storage/vitest.config.ts | 7 + pnpm-lock.yaml | 19 + 34 files changed, 1187 insertions(+), 10 deletions(-) create mode 100644 apps/api/scripts/storage-reconcile.ts create mode 100644 apps/api/src/plugins/storage.ts create mode 100644 apps/api/src/routes/storage.test.ts create mode 100644 apps/api/src/routes/storage.ts create mode 100644 apps/api/src/storage/factory.ts create mode 100644 apps/api/src/storage/reconcile.test.ts create mode 100644 apps/api/src/storage/reconcile.ts create mode 100644 packages/db/migrations/0007_files_checksum_versioning.sql create mode 100644 packages/storage/package.json create mode 100644 packages/storage/src/driver.ts create mode 100644 packages/storage/src/fs-driver.test.ts create mode 100644 packages/storage/src/fs-driver.ts create mode 100644 packages/storage/src/index.ts create mode 100644 packages/storage/src/validation.test.ts create mode 100644 packages/storage/src/validation.ts create mode 100644 packages/storage/tsconfig.json create mode 100644 packages/storage/vitest.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6eef815..671b25c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,9 @@ jobs: - name: Unit tests (@appbase/sdk) run: pnpm --filter @appbase/sdk test + - name: Unit tests (@appbase/storage) + run: pnpm --filter @appbase/storage test + - name: Unit & integration tests (api) run: pnpm --filter api test diff --git a/apps/api/.env.example b/apps/api/.env.example index 5e78370..599995e 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -2,6 +2,11 @@ NODE_ENV=development HOST=localhost PORT=3000 DB_PATH=data/appbase.sqlite +# Object bytes root (dev: defaults to ./data/storage; production unset default: /app/data/storage) +# STORAGE_ROOT=data/storage +# STORAGE_MAX_UPLOAD_BYTES=52428800 +# STORAGE_ALLOWED_MIME=image/*,application/pdf,text/* +# STORAGE_DRIVER=fs LOG_LEVEL=info BASE_URL=http://localhost:3000 # Comma-separated browser origins allowed for credentialed CORS (e.g. your Next.js dev URL). diff --git a/apps/api/README.md b/apps/api/README.md index 3363be8..67dd1d0 100644 --- a/apps/api/README.md +++ b/apps/api/README.md @@ -10,7 +10,7 @@ In M1, this service is the core runtime for: - admin-facing management endpoints - OpenAPI documentation -At the moment, this package is still in the scaffolding phase. The infrastructure is in place, but only the `/health` route is implemented. +Core routes include `/health`, `/auth/*`, `/db/*`, `/storage/*`, and OpenAPI `/docs`. ## Current Responsibilities @@ -38,9 +38,15 @@ src/ │ └── not-found.ts # 404 handling ├── plugins/ │ ├── database.ts # DB decoration and initialization -│ └── infrastructure.ts # CORS, multipart, Swagger, Swagger UI +│ ├── auth.ts # better-auth integration +│ ├── infrastructure.ts # CORS, multipart, Swagger, Swagger UI +│ └── storage.ts # StorageDriver + volume readiness +├── storage/ # API wiring: factory, reconcile (shared driver: `@appbase/storage`) ├── routes/ │ ├── health.ts # /health route + OpenAPI schema +│ ├── auth.ts +│ ├── db.ts +│ ├── storage.ts │ └── index.ts ├── types/ │ └── fastify.d.ts # Fastify instance decorations @@ -55,6 +61,7 @@ Default runtime values: - `HOST=0.0.0.0` - `PORT=3000` - `DB_PATH=data/appbase.sqlite` +- `STORAGE_ROOT` — development default `./data/storage`; production default `/app/data/storage` (use a volume) - `LOG_LEVEL=info` Optional / reserved values: diff --git a/apps/api/data/appbase.sqlite-shm b/apps/api/data/appbase.sqlite-shm index acadd027b56d830e12e8581d34cce79e34065f4f..56db84707f7b542c26d24578f2d21c41d16484a5 100644 GIT binary patch delta 222 zcmZo@U}|V!s+V}A%K!t63=9GmKtdWQplqD?G9f&AgNRJt$B9Yi?a|No{_;v`$|qGl z%xsXk|B(PxoQc6=W8-xeUZ5-k6A&{4vE=4N)_j}IR~!u(8BHe(I=!7-!1`eGB^NzL dMl&e)(dG;8K(0BA%PhlS0cBfl{^8AG0ssdPL+}6q delta 192 zcmZo@U}|V!s+V}A%K!t63=9G$KtdWQP+gzqlsgXg7v59GFl2KBMiAAbmilI>| z<6N%AXExXP=`bnOmZ~hzowMNs7sy`zn+*Ja_`mai;(yEkg8wo9J^q_Odk^xnvM@6+ za!l?^P*s%#s*{F!?`58`@`mW}1YYjV4BTw)4E)D=gL#$M+_zuY#AwdO)od-uE-o(4 z*cQ6Ib{``*qpLzfaz<)$c5!KLfHqdJdNHX>O}4jH-~MhFV=FT-a2fNr7cOVa MX59X0KjS?n0LCbWHUIzs delta 46 zcmaE`%dcUVZ$k@X3sVbo3rh=Y3tJ0&3r7oQ3s(zw3r`Dg3ttO=i$IHDi_j9`SO9~~ B4)*{6 diff --git a/apps/api/package.json b/apps/api/package.json index 4a924e7..6d12025 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -11,10 +11,12 @@ "check-types": "tsc --noEmit", "lint": "eslint src", "test": "vitest run", - "test:watch": "vitest" + "test:watch": "vitest", + "storage:reconcile": "tsx scripts/storage-reconcile.ts" }, "dependencies": { "@appbase/db": "workspace:*", + "@appbase/storage": "workspace:*", "@appbase/types": "workspace:*", "drizzle-orm": "^0.45.1", "@better-auth/api-key": "^1.5.5", diff --git a/apps/api/scripts/storage-reconcile.ts b/apps/api/scripts/storage-reconcile.ts new file mode 100644 index 0000000..840929f --- /dev/null +++ b/apps/api/scripts/storage-reconcile.ts @@ -0,0 +1,22 @@ +/** + * Operator helper: list metadata↔disk drift (see docs/STORAGE-OPERATIONS.md). + * Usage: from repo root, `pnpm --filter api exec tsx scripts/storage-reconcile.ts` + * (with `.env` or env vars for DB_PATH, STORAGE_ROOT). + */ +import { config as loadDotenv } from "dotenv"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { createDb } from "@appbase/db"; +import { loadEnv } from "../src/config/env"; +import { createStorageDriver } from "../src/storage/factory"; +import { reconcileFileStorage } from "../src/storage/reconcile"; + +const dir = path.dirname(fileURLToPath(import.meta.url)); +loadDotenv({ path: path.resolve(dir, "../.env"), quiet: true }); + +const env = loadEnv(process.env); +const db = createDb(env.DB_PATH); +const driver = createStorageDriver(env); +const report = await reconcileFileStorage(db, driver); +console.log(JSON.stringify(report, null, 2)); +db.$client.close(); diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 081e4a0..a308a57 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -5,6 +5,7 @@ import { registerMiddleware } from "./middleware"; import { registerDatabase } from "./plugins/database"; import { registerInfrastructure } from "./plugins/infrastructure"; import { registerAuth } from "./plugins/auth"; +import { registerStorage } from "./plugins/storage"; import { registerRoutes } from "./routes"; export interface BuildAppOptions { @@ -21,6 +22,7 @@ export async function buildApp({ env }: BuildAppOptions): Promise/data/storage`. */ + STORAGE_ROOT: z.string().optional(), + /** Max upload size in bytes (multipart / storage enforcement). Default 50 MiB. */ + STORAGE_MAX_UPLOAD_BYTES: z.coerce.number().int().positive().optional(), + /** + * Comma-separated MIME types and `type/*` patterns. Empty or `*` = allow all (dev-friendly). + * Example: `image/*,application/pdf,text/plain` + */ + STORAGE_ALLOWED_MIME: z.string().optional(), + /** `fs` only in M1; reserved for future `s3` driver. */ + STORAGE_DRIVER: z.enum(["fs"]).default("fs"), LOG_LEVEL: z .enum(["fatal", "error", "warn", "info", "debug", "trace", "silent"]) .default("info"), @@ -19,11 +33,21 @@ const envSchema = z.object({ type ParsedEnv = z.output; -export type AppEnv = Omit & { +export type StorageEnv = { + storageRoot: string; + storageMaxUploadBytes: number; + storageAllowedMime: string | null; + storageDriver: "fs"; +}; + +export type AppEnv = Omit< + ParsedEnv, + "BASE_URL" | "AUTH_SECRET" | "CORS_ORIGINS" | "STORAGE_ROOT" | "STORAGE_MAX_UPLOAD_BYTES" | "STORAGE_ALLOWED_MIME" +> & { BASE_URL: string; AUTH_SECRET: string; corsAllowedOrigins: string[]; -}; +} & StorageEnv; function buildCorsAllowedOrigins(corsOriginsEnv: string | undefined, baseUrl: string): string[] { const fromEnv = @@ -61,12 +85,33 @@ export function loadEnv(source: NodeJS.ProcessEnv): AppEnv { } const baseUrl = parsed.BASE_URL ?? `http://${publicHost}:${parsed.PORT}`; - const { CORS_ORIGINS, ...rest } = parsed; + const { CORS_ORIGINS, STORAGE_ROOT, STORAGE_MAX_UPLOAD_BYTES, STORAGE_ALLOWED_MIME, ...rest } = parsed; + + const storageRoot = + STORAGE_ROOT && STORAGE_ROOT.length > 0 + ? path.isAbsolute(STORAGE_ROOT) + ? STORAGE_ROOT + : path.resolve(process.cwd(), STORAGE_ROOT) + : parsed.NODE_ENV === "production" + ? "/app/data/storage" + : parsed.NODE_ENV === "test" + ? path.join(os.tmpdir(), `appbase-api-storage-${randomUUID()}`) + : path.resolve(process.cwd(), "data/storage"); + + const storageMaxUploadBytes = STORAGE_MAX_UPLOAD_BYTES ?? 50 * 1024 * 1024; + const storageAllowedMime = + STORAGE_ALLOWED_MIME == null || STORAGE_ALLOWED_MIME.trim() === "" || STORAGE_ALLOWED_MIME.trim() === "*" + ? null + : STORAGE_ALLOWED_MIME.trim(); return { ...rest, BASE_URL: baseUrl, AUTH_SECRET: authSecret ?? "dev-secret-min-32-chars-required-for-auth", corsAllowedOrigins: buildCorsAllowedOrigins(CORS_ORIGINS, baseUrl), + storageRoot, + storageMaxUploadBytes, + storageAllowedMime, + storageDriver: parsed.STORAGE_DRIVER, }; } diff --git a/apps/api/src/constants.ts b/apps/api/src/constants.ts index 2d0d36e..75e10e4 100644 --- a/apps/api/src/constants.ts +++ b/apps/api/src/constants.ts @@ -46,8 +46,8 @@ export const TEST_EXCLUDED_AUTH_POST_PATHS: readonly string[] = [ "/auth/logout", ]; -/** In test: /db/* skips x-api-key so db integration tests run with JWT only. */ -export const TEST_EXCLUDED_API_KEY_PATHS: readonly string[] = ["/db/"]; +/** In test: /db/* and /storage/* skip x-api-key so integration tests run with JWT only. */ +export const TEST_EXCLUDED_API_KEY_PATHS: readonly string[] = ["/db/", "/storage/"]; /** URL path prefixes that require a verified JWT access token. */ diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 598a284..b162ae8 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -30,7 +30,7 @@ async function main() { await app.listen({ host: env.HOST, port: env.PORT }); app.log.info( - { host: env.HOST, port: env.PORT, dbPath: env.DB_PATH }, + { host: env.HOST, port: env.PORT, dbPath: env.DB_PATH, storageRoot: env.storageRoot }, "AppBase API started", ); } diff --git a/apps/api/src/plugins/infrastructure.ts b/apps/api/src/plugins/infrastructure.ts index 8d8593a..0cc78d4 100644 --- a/apps/api/src/plugins/infrastructure.ts +++ b/apps/api/src/plugins/infrastructure.ts @@ -24,7 +24,7 @@ export async function registerInfrastructure(app: FastifyInstance, env: AppEnv) cb(new Error("CORS origin not allowed"), false); }, }); - await app.register(multipart, { limits: { fileSize: 50 * 1024 * 1024 } }); + await app.register(multipart, { limits: { fileSize: env.storageMaxUploadBytes } }); await app.register(swagger, { openapi: { info: { @@ -35,6 +35,7 @@ export async function registerInfrastructure(app: FastifyInstance, env: AppEnv) tags: [ { name: "auth", description: "Registration, login, session, and tokens." }, { name: "database", description: "User-scoped collections and records (Bearer JWT)." }, + { name: "storage", description: "User-scoped file uploads and downloads (Bearer JWT)." }, { name: "system", description: "Operational and health endpoints." }, ], components: { diff --git a/apps/api/src/plugins/storage.ts b/apps/api/src/plugins/storage.ts new file mode 100644 index 0000000..427bdcb --- /dev/null +++ b/apps/api/src/plugins/storage.ts @@ -0,0 +1,26 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { FastifyInstance } from "fastify"; +import type { AppEnv } from "../config/env"; +import { createStorageDriver } from "../storage/factory"; + +export async function registerStorage(app: FastifyInstance, env: AppEnv) { + fs.mkdirSync(env.storageRoot, { recursive: true }); + const probe = path.join(env.storageRoot, ".write-check"); + try { + fs.writeFileSync(probe, "ok"); + fs.unlinkSync(probe); + } catch { + throw new Error(`Storage root is not writable: ${env.storageRoot}`); + } + + if (env.NODE_ENV === "production" && !env.storageRoot.startsWith("/app/data")) { + app.log.warn( + { storageRoot: env.storageRoot }, + "STORAGE_ROOT is outside /app/data; ensure a persistent volume is mounted. See docs/adr/ADR-005-file-storage-strategy.md.", + ); + } + + const driver = createStorageDriver(env); + app.decorate("storageDriver", driver); +} diff --git a/apps/api/src/routes/index.ts b/apps/api/src/routes/index.ts index 3599f5f..5129477 100644 --- a/apps/api/src/routes/index.ts +++ b/apps/api/src/routes/index.ts @@ -2,9 +2,11 @@ import type { FastifyInstance } from "fastify"; import { registerHealthRoutes } from "./health"; import { registerAuthRoutes } from "./auth"; import { registerDbRoutes } from "./db"; +import { registerStorageRoutes } from "./storage"; export async function registerRoutes(app: FastifyInstance) { await registerHealthRoutes(app); await registerAuthRoutes(app); await registerDbRoutes(app); + await registerStorageRoutes(app); } diff --git a/apps/api/src/routes/storage.test.ts b/apps/api/src/routes/storage.test.ts new file mode 100644 index 0000000..8d19f78 --- /dev/null +++ b/apps/api/src/routes/storage.test.ts @@ -0,0 +1,280 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, beforeAll, afterAll, beforeEach } from "vitest"; +import { buildApp } from "../app"; +import { loadEnv } from "../config/env"; +import { reconcileFileStorage } from "../storage/reconcile"; + +const TEST_PASSWORD = "SecurePassword123!"; + +function multipartUpload(filename: string, content: string, mime = "text/plain"): { + payload: string; + headers: Record; +} { + const boundary = "----AppBaseStorageTest"; + const payload = + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="file"; filename="${filename}"\r\n` + + `Content-Type: ${mime}\r\n\r\n` + + `${content}\r\n` + + `--${boundary}--\r\n`; + return { + payload, + headers: { "Content-Type": `multipart/form-data; boundary=${boundary}` }, + }; +} + +async function createTestUser(app: Awaited>): Promise<{ + accessToken: string; + userId: string; + email: string; +}> { + const email = `st-${Date.now()}-${Math.random().toString(36).slice(2)}@example.com`; + const registerRes = await app.inject({ + method: "POST", + url: "/auth/register", + payload: { email, password: TEST_PASSWORD }, + }); + expect(registerRes.statusCode).toBe(201); + const { data } = registerRes.json() as { data: { accessToken: string; user: { id: string } } }; + return { accessToken: data.accessToken, userId: data.user.id, email }; +} + +describe("storage routes", () => { + const port = 39572 + (Math.floor(Math.random() * 500) % 500); + const baseUrl = `http://127.0.0.1:${port}`; + + let tmpDir: string; + let dbPath: string; + let storageRoot: string; + let testEnv: ReturnType; + let app: Awaited>; + let accessToken: string; + let userEmail: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "appbase-api-storage-")); + dbPath = path.join(tmpDir, "app.sqlite"); + storageRoot = path.join(tmpDir, "storage"); + testEnv = loadEnv({ + ...process.env, + NODE_ENV: "test", + DB_PATH: dbPath, + STORAGE_ROOT: storageRoot, + AUTH_SECRET: "test-secret-must-be-at-least-32-characters-long", + BASE_URL: baseUrl, + STORAGE_ALLOWED_MIME: "*", + }); + app = await buildApp({ env: testEnv }); + await app.listen({ port, host: "127.0.0.1" }); + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + const u = await createTestUser(app); + accessToken = u.accessToken; + userEmail = u.email; + }); + + const authHeaders = (extra?: Record) => ({ + Authorization: `Bearer ${accessToken}`, + ...extra, + }); + + it("POST upload → GET list → GET download → DELETE roundtrip", async () => { + const mp = multipartUpload("hello.txt", "roundtrip-body", "text/plain"); + const up = await app.inject({ + method: "POST", + url: "/storage/buckets/docs/upload", + headers: { ...authHeaders(), ...mp.headers }, + payload: mp.payload, + }); + expect(up.statusCode).toBe(201); + const upJson = up.json() as { data: { file: { id: string }; url: string } }; + const fileId = upJson.data.file.id; + expect(upJson.data.url).toBe(`/storage/buckets/docs/${fileId}`); + + const list = await app.inject({ + method: "GET", + url: "/storage/buckets/docs", + headers: authHeaders(), + }); + expect(list.statusCode).toBe(200); + const listJson = list.json() as { data: { files: { id: string }[]; total: number } }; + expect(listJson.data.total).toBe(1); + expect(listJson.data.files[0]!.id).toBe(fileId); + + const dl = await app.inject({ + method: "GET", + url: `/storage/buckets/docs/${fileId}`, + headers: authHeaders(), + }); + expect(dl.statusCode).toBe(200); + expect(dl.body).toBe("roundtrip-body"); + expect(dl.headers["content-type"]).toContain("text/plain"); + + const del = await app.inject({ + method: "DELETE", + url: `/storage/buckets/docs/${fileId}`, + headers: authHeaders(), + }); + expect(del.statusCode).toBe(200); + + const list2 = await app.inject({ + method: "GET", + url: "/storage/buckets/docs", + headers: authHeaders(), + }); + expect((list2.json() as { data: { total: number } }).data.total).toBe(0); + }); + + it("isolates ownership: other user cannot download", async () => { + const mp = multipartUpload("secret.txt", "x", "text/plain"); + const up = await app.inject({ + method: "POST", + url: "/storage/buckets/private/upload", + headers: { ...authHeaders(), ...mp.headers }, + payload: mp.payload, + }); + expect(up.statusCode).toBe(201); + const fileId = (up.json() as { data: { file: { id: string } } }).data.file.id; + + const other = await createTestUser(app); + const probe = await app.inject({ + method: "GET", + url: `/storage/buckets/private/${fileId}`, + headers: { Authorization: `Bearer ${other.accessToken}` }, + }); + expect(probe.statusCode).toBe(404); + }); + + it("rejects invalid bucket names", async () => { + const mp = multipartUpload("a.txt", "x"); + const res = await app.inject({ + method: "POST", + url: "/storage/buckets/not%20valid!/upload", + headers: { ...authHeaders(), ...mp.headers }, + payload: mp.payload, + }); + expect(res.statusCode).toBe(400); + }); + + it("rejects disallowed MIME when policy is set", async () => { + const narrowPort = port + 10_000; + const narrowBase = `http://127.0.0.1:${narrowPort}`; + const narrowEnv = loadEnv({ + ...process.env, + NODE_ENV: "test", + DB_PATH: path.join(tmpDir, `narrow-${Date.now()}.sqlite`), + STORAGE_ROOT: path.join(tmpDir, `narrow-storage-${Date.now()}`), + AUTH_SECRET: "test-secret-must-be-at-least-32-characters-long", + BASE_URL: narrowBase, + STORAGE_ALLOWED_MIME: "image/png", + }); + const app2 = await buildApp({ env: narrowEnv }); + await app2.listen({ port: narrowPort, host: "127.0.0.1" }); + const u = await createTestUser(app2); + const mp = multipartUpload("x.txt", "hi", "text/plain"); + const res = await app2.inject({ + method: "POST", + url: "/storage/buckets/b/upload", + headers: { Authorization: `Bearer ${u.accessToken}`, ...mp.headers }, + payload: mp.payload, + }); + expect(res.statusCode).toBe(400); + await app2.close(); + }); + + it("returns 404 when object bytes are missing on disk", async () => { + const mp = multipartUpload("gone.bin", "data", "application/octet-stream"); + const up = await app.inject({ + method: "POST", + url: "/storage/buckets/b/upload", + headers: { ...authHeaders(), ...mp.headers }, + payload: mp.payload, + }); + expect(up.statusCode).toBe(201); + const fileId = (up.json() as { data: { file: { id: string } } }).data.file.id; + + const objectPath = path.join(storageRoot, "objects", fileId); + expect(fs.existsSync(objectPath)).toBe(true); + fs.unlinkSync(objectPath); + + const dl = await app.inject({ + method: "GET", + url: `/storage/buckets/b/${fileId}`, + headers: authHeaders(), + }); + expect(dl.statusCode).toBe(404); + }); + + it("survives process restart with same DB and storage paths (login + download)", async () => { + const mp = multipartUpload("persist.txt", " durable ", "text/plain"); + const up = await app.inject({ + method: "POST", + url: "/storage/buckets/keep/upload", + headers: { ...authHeaders(), ...mp.headers }, + payload: mp.payload, + }); + expect(up.statusCode).toBe(201); + const fileId = (up.json() as { data: { file: { id: string } } }).data.file.id; + + await app.close(); + + const app2 = await buildApp({ env: testEnv }); + await app2.listen({ port, host: "127.0.0.1" }); + const login = await app2.inject({ + method: "POST", + url: "/auth/login", + payload: { email: userEmail, password: TEST_PASSWORD }, + }); + expect(login.statusCode).toBe(200); + const token2 = (login.json() as { data: { accessToken: string } }).data.accessToken; + + const dl = await app2.inject({ + method: "GET", + url: `/storage/buckets/keep/${fileId}`, + headers: { Authorization: `Bearer ${token2}` }, + }); + expect(dl.statusCode).toBe(200); + expect(dl.body).toBe(" durable "); + + await app2.close(); + app = await buildApp({ env: testEnv }); + await app.listen({ port, host: "127.0.0.1" }); + }); +}); + +describe("storage reconciliation (integration)", () => { + it("reconcile sees uploaded file in metadata and on disk", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "appbase-st-recon-")); + const dbFile = path.join(tmp, "r.sqlite"); + const sRoot = path.join(tmp, "s"); + const env = loadEnv({ + ...process.env, + NODE_ENV: "test", + DB_PATH: dbFile, + STORAGE_ROOT: sRoot, + AUTH_SECRET: "test-secret-must-be-at-least-32-characters-long", + BASE_URL: "http://127.0.0.1:9", + STORAGE_ALLOWED_MIME: "*", + }); + const application = await buildApp({ env }); + const u = await createTestUser(application); + const mp = multipartUpload("x.bin", "y"); + await application.inject({ + method: "POST", + url: "/storage/buckets/z/upload", + headers: { Authorization: `Bearer ${u.accessToken}`, ...mp.headers }, + payload: mp.payload, + }); + const r = await reconcileFileStorage(application.db, application.storageDriver); + expect(r.metadataMissingObject).toEqual([]); + expect(r.objectWithoutMetadata).toEqual([]); + await application.close(); + }); +}); diff --git a/apps/api/src/routes/storage.ts b/apps/api/src/routes/storage.ts new file mode 100644 index 0000000..5d51d73 --- /dev/null +++ b/apps/api/src/routes/storage.ts @@ -0,0 +1,342 @@ +import type { FastifyInstance } from "fastify"; +import { nanoid } from "nanoid"; +import { eq, and, count } from "drizzle-orm"; +import { files } from "@appbase/db/schema"; +import { validateBucket, isMimeAllowed } from "@appbase/storage"; + +function apiSuccess(data: T) { + return { success: true as const, data }; +} + +function apiError(code: string, message: string) { + return { success: false as const, error: { code, message } }; +} + +const FILE_ID_REGEX = /^[a-zA-Z0-9_-]{8,64}$/; + +function toIso(d: Date): string { + return d instanceof Date ? d.toISOString() : String(d); +} + +function toFileRecord(row: typeof files.$inferSelect) { + return { + id: row.id, + bucket: row.bucket, + filename: row.filename, + mimeType: row.mimeType, + size: row.size, + ownerId: row.ownerId, + createdAt: toIso(row.createdAt as unknown as Date), + }; +} + +const uploadOpenApi = { + tags: ["storage"], + summary: "Upload file", + description: "Multipart upload (`file` field) into a user-scoped bucket.", + security: [{ bearerAuth: [] }, { apiKey: [] }], + consumes: ["multipart/form-data"], + params: { + type: "object", + required: ["bucket"], + properties: { + bucket: { type: "string", description: "Bucket name (letters, digits, hyphens, underscores; max 64)." }, + }, + }, + response: { + 201: { + type: "object", + properties: { + success: { type: "boolean", const: true }, + data: { + type: "object", + properties: { + file: { type: "object", additionalProperties: true }, + url: { type: "string" }, + }, + required: ["file", "url"], + }, + }, + required: ["success", "data"], + }, + 400: { type: "object", additionalProperties: true }, + 401: { type: "object", additionalProperties: true }, + 413: { type: "object", additionalProperties: true }, + }, +} as const; + +const listOpenApi = { + tags: ["storage"], + summary: "List files in bucket", + security: [{ bearerAuth: [] }, { apiKey: [] }], + params: { + type: "object", + required: ["bucket"], + properties: { bucket: { type: "string" } }, + }, + response: { + 200: { + type: "object", + properties: { + success: { type: "boolean", const: true }, + data: { + type: "object", + properties: { + files: { type: "array", items: { type: "object", additionalProperties: true } }, + total: { type: "integer" }, + }, + required: ["files", "total"], + }, + }, + required: ["success", "data"], + }, + 400: { type: "object", additionalProperties: true }, + 401: { type: "object", additionalProperties: true }, + }, +} as const; + +const downloadOpenApi = { + tags: ["storage"], + summary: "Download file", + security: [{ bearerAuth: [] }, { apiKey: [] }], + params: { + type: "object", + required: ["bucket", "fileId"], + properties: { + bucket: { type: "string" }, + fileId: { type: "string" }, + }, + }, + response: { + 200: { description: "Binary body", type: "string", format: "binary" }, + 400: { type: "object", additionalProperties: true }, + 401: { type: "object", additionalProperties: true }, + 404: { type: "object", additionalProperties: true }, + }, +} as const; + +const deleteOpenApi = { + tags: ["storage"], + summary: "Delete file", + security: [{ bearerAuth: [] }, { apiKey: [] }], + params: { + type: "object", + required: ["bucket", "fileId"], + properties: { + bucket: { type: "string" }, + fileId: { type: "string" }, + }, + }, + response: { + 200: { + type: "object", + properties: { + success: { type: "boolean", const: true }, + data: { + type: "object", + properties: { deleted: { type: "boolean", const: true } }, + required: ["deleted"], + }, + }, + required: ["success", "data"], + }, + 400: { type: "object", additionalProperties: true }, + 401: { type: "object", additionalProperties: true }, + 404: { type: "object", additionalProperties: true }, + }, +} as const; + +export async function registerStorageRoutes(app: FastifyInstance) { + const driver = app.storageDriver; + const cfg = app.config; + + // POST /storage/buckets/:bucket/upload + app.post<{ + Params: { bucket: string }; + }>("/storage/buckets/:bucket/upload", { schema: uploadOpenApi }, async (request, reply) => { + const userId = request.userId; + if (!userId) { + return reply.status(401).send(apiError("INVALID_TOKEN", "The provided access token is invalid or expired.")); + } + + const { bucket } = request.params; + const bucketCheck = validateBucket(bucket); + if (!bucketCheck.valid) { + return reply.status(400).send(apiError("VALIDATION_ERROR", bucketCheck.error)); + } + + const file = await request.file(); + if (!file) { + return reply.status(400).send(apiError("VALIDATION_ERROR", "Multipart field `file` is required")); + } + if (file.fieldname !== "file") { + return reply.status(400).send(apiError("VALIDATION_ERROR", "Multipart field must be named `file`")); + } + + const mimeType = file.mimetype || "application/octet-stream"; + if (!isMimeAllowed(mimeType, cfg.storageAllowedMime)) { + return reply.status(400).send(apiError("VALIDATION_ERROR", "MIME type is not allowed by server policy")); + } + + const filename = file.filename || "upload"; + const id = nanoid(); + const storagePath = `objects/${id}`; + + let putResult: { size: number; checksum: string }; + try { + putResult = await driver.putObject({ objectKey: storagePath, stream: file.file }); + } catch (err) { + request.log.error({ err }, "storage.upload.putObject failed"); + return reply.status(500).send(apiError("INTERNAL_ERROR", "Failed to store file")); + } + + if (putResult.size > cfg.storageMaxUploadBytes) { + await driver.deleteObject(storagePath).catch(() => {}); + return reply.status(413).send(apiError("PAYLOAD_TOO_LARGE", "File exceeds maximum upload size")); + } + + const now = new Date(); + await app.db.insert(files).values({ + id, + logicalFileId: id, + version: 1, + bucket, + filename, + mimeType, + size: putResult.size, + storagePath, + checksum: putResult.checksum, + ownerId: userId, + createdAt: now, + }); + + const url = `/storage/buckets/${bucket}/${id}`; + request.log.info({ bucket, id, size: putResult.size, userId }, "storage.upload"); + + return reply.status(201).send( + apiSuccess({ + file: { + id, + bucket, + filename, + mimeType, + size: putResult.size, + ownerId: userId, + createdAt: now.toISOString(), + }, + url, + }), + ); + }); + + // GET /storage/buckets/:bucket + app.get<{ Params: { bucket: string } }>("/storage/buckets/:bucket", { schema: listOpenApi }, async (request, reply) => { + const userId = request.userId; + if (!userId) { + return reply.status(401).send(apiError("INVALID_TOKEN", "The provided access token is invalid or expired.")); + } + + const { bucket } = request.params; + const bucketCheck = validateBucket(bucket); + if (!bucketCheck.valid) { + return reply.status(400).send(apiError("VALIDATION_ERROR", bucketCheck.error)); + } + + const [rows, totalResult] = await Promise.all([ + app.db + .select() + .from(files) + .where(and(eq(files.bucket, bucket), eq(files.ownerId, userId))) + .orderBy(files.createdAt), + app.db + .select({ count: count() }) + .from(files) + .where(and(eq(files.bucket, bucket), eq(files.ownerId, userId))), + ]); + + const total = totalResult[0]?.count ?? 0; + request.log.info({ bucket, total, userId }, "storage.list"); + + return reply.send(apiSuccess({ files: rows.map(toFileRecord), total })); + }); + + // GET /storage/buckets/:bucket/:fileId + app.get<{ Params: { bucket: string; fileId: string } }>( + "/storage/buckets/:bucket/:fileId", + { schema: downloadOpenApi }, + async (request, reply) => { + const userId = request.userId; + if (!userId) { + return reply.status(401).send(apiError("INVALID_TOKEN", "The provided access token is invalid or expired.")); + } + + const { bucket, fileId } = request.params; + const bucketCheck = validateBucket(bucket); + if (!bucketCheck.valid) { + return reply.status(400).send(apiError("VALIDATION_ERROR", bucketCheck.error)); + } + if (!FILE_ID_REGEX.test(fileId)) { + return reply.status(400).send(apiError("VALIDATION_ERROR", "Invalid file id")); + } + + const rows = await app.db + .select() + .from(files) + .where(and(eq(files.id, fileId), eq(files.bucket, bucket), eq(files.ownerId, userId))) + .limit(1); + + if (rows.length === 0) { + return reply.status(404).send(apiError("NOT_FOUND", "File not found")); + } + + const row = rows[0]!; + if (!(await driver.exists(row.storagePath))) { + return reply.status(404).send(apiError("NOT_FOUND", "File data is missing")); + } + + request.log.info({ bucket, fileId, userId }, "storage.download"); + const stream = driver.getObjectStream(row.storagePath); + return reply.header("Content-Type", row.mimeType).send(stream); + }, + ); + + // DELETE /storage/buckets/:bucket/:fileId + app.delete<{ Params: { bucket: string; fileId: string } }>( + "/storage/buckets/:bucket/:fileId", + { schema: deleteOpenApi }, + async (request, reply) => { + const userId = request.userId; + if (!userId) { + return reply.status(401).send(apiError("INVALID_TOKEN", "The provided access token is invalid or expired.")); + } + + const { bucket, fileId } = request.params; + const bucketCheck = validateBucket(bucket); + if (!bucketCheck.valid) { + return reply.status(400).send(apiError("VALIDATION_ERROR", bucketCheck.error)); + } + if (!FILE_ID_REGEX.test(fileId)) { + return reply.status(400).send(apiError("VALIDATION_ERROR", "Invalid file id")); + } + + const rows = await app.db + .select() + .from(files) + .where(and(eq(files.id, fileId), eq(files.bucket, bucket), eq(files.ownerId, userId))) + .limit(1); + + if (rows.length === 0) { + return reply.status(404).send(apiError("NOT_FOUND", "File not found")); + } + + const row = rows[0]!; + await app.db.delete(files).where(and(eq(files.id, fileId), eq(files.ownerId, userId))); + await driver.deleteObject(row.storagePath).catch((err) => { + request.log.warn({ err, storagePath: row.storagePath }, "storage.delete.object_cleanup_failed"); + }); + + request.log.info({ bucket, fileId, userId }, "storage.delete"); + return reply.send(apiSuccess({ deleted: true })); + }, + ); +} diff --git a/apps/api/src/storage/factory.ts b/apps/api/src/storage/factory.ts new file mode 100644 index 0000000..369809d --- /dev/null +++ b/apps/api/src/storage/factory.ts @@ -0,0 +1,9 @@ +import { FileSystemStorageDriver, type StorageDriver } from "@appbase/storage"; +import type { AppEnv } from "../config/env"; + +export function createStorageDriver(env: AppEnv): StorageDriver { + if (env.storageDriver !== "fs") { + throw new Error(`Unsupported STORAGE_DRIVER: ${env.storageDriver}`); + } + return new FileSystemStorageDriver(env.storageRoot); +} diff --git a/apps/api/src/storage/reconcile.test.ts b/apps/api/src/storage/reconcile.test.ts new file mode 100644 index 0000000..fcf2fbf --- /dev/null +++ b/apps/api/src/storage/reconcile.test.ts @@ -0,0 +1,78 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { Readable } from "node:stream"; +import { createDb } from "@appbase/db"; +import { files, user } from "@appbase/db/schema"; +import { FileSystemStorageDriver } from "@appbase/storage"; +import { reconcileFileStorage } from "./reconcile"; + +describe("reconcileFileStorage", () => { + let tmp: string; + let db: ReturnType; + let driver: FileSystemStorageDriver; + + beforeEach(async () => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), "appbase-recon-")); + const dbPath = path.join(tmp, "test.sqlite"); + db = createDb(dbPath); + driver = new FileSystemStorageDriver(path.join(tmp, "blobs")); + await db.insert(user).values({ + id: "user1", + email: `recon-${Date.now()}-${Math.random()}@t.com`, + emailVerified: true, + createdAt: new Date(), + updatedAt: new Date(), + }); + }); + + afterEach(() => { + db.$client.close(); + }); + + it("reports metadata rows whose object is missing on disk", async () => { + await db.insert(files).values({ + id: "file_meta_only", + logicalFileId: "file_meta_only", + version: 1, + bucket: "b", + filename: "x.txt", + mimeType: "text/plain", + size: 1, + storagePath: "objects/ghost", + ownerId: "user1", + createdAt: new Date(), + }); + + const report = await reconcileFileStorage(db, driver); + expect(report.metadataMissingObject).toContain("file_meta_only"); + expect(report.objectWithoutMetadata).toEqual([]); + }); + + it("reports on-disk objects not referenced by metadata", async () => { + fs.mkdirSync(path.join(tmp, "blobs", "objects"), { recursive: true }); + fs.writeFileSync(path.join(tmp, "blobs", "objects", "orphan"), "data"); + const report = await reconcileFileStorage(db, driver); + expect(report.objectWithoutMetadata).toContain("objects/orphan"); + }); + + it("is consistent when row and bytes exist", async () => { + await driver.putObject({ objectKey: "objects/real1", stream: Readable.from(Buffer.from("ok")) }); + await db.insert(files).values({ + id: "real1", + logicalFileId: "real1", + version: 1, + bucket: "b", + filename: "f.bin", + mimeType: "application/octet-stream", + size: 2, + storagePath: "objects/real1", + ownerId: "user1", + createdAt: new Date(), + }); + const report = await reconcileFileStorage(db, driver); + expect(report.metadataMissingObject).toEqual([]); + expect(report.objectWithoutMetadata).toEqual([]); + }); +}); diff --git a/apps/api/src/storage/reconcile.ts b/apps/api/src/storage/reconcile.ts new file mode 100644 index 0000000..75a48ee --- /dev/null +++ b/apps/api/src/storage/reconcile.ts @@ -0,0 +1,34 @@ +import { files } from "@appbase/db/schema"; +import type { AppDb } from "@appbase/db"; +import { FileSystemStorageDriver, type StorageDriver } from "@appbase/storage"; + +export type ReconcileReport = { + /** `files.id` rows whose object key is missing on disk */ + metadataMissingObject: string[]; + /** Relative object keys present on disk but not referenced by any `files.storage_path` */ + objectWithoutMetadata: string[]; +}; + +/** + * Detects inconsistent storage vs metadata. FS driver: compares DB `storage_path` with files under `objects/`. + * For future S3 adapters, replace disk scan with bucket listing or admin tooling. + */ +export async function reconcileFileStorage(db: AppDb, driver: StorageDriver): Promise { + const rows = await db.select({ id: files.id, storagePath: files.storagePath }).from(files); + const pathSet = new Set(rows.map((r) => r.storagePath)); + + const metadataMissingObject: string[] = []; + for (const row of rows) { + if (!(await driver.exists(row.storagePath))) { + metadataMissingObject.push(row.id); + } + } + + let objectWithoutMetadata: string[] = []; + if (driver instanceof FileSystemStorageDriver) { + const onDisk = await driver.listStoredObjectKeys(); + objectWithoutMetadata = onDisk.filter((k) => !pathSet.has(k)); + } + + return { metadataMissingObject, objectWithoutMetadata }; +} diff --git a/apps/api/src/types/fastify.d.ts b/apps/api/src/types/fastify.d.ts index a5f9239..6105733 100644 --- a/apps/api/src/types/fastify.d.ts +++ b/apps/api/src/types/fastify.d.ts @@ -1,12 +1,14 @@ import type { AppDb } from "@appbase/db"; import type { AppEnv } from "../config/env"; import type { Auth } from "../lib/auth"; +import type { StorageDriver } from "@appbase/storage"; declare module "fastify" { interface FastifyInstance { db: AppDb; config: AppEnv; auth: Auth; + storageDriver: StorageDriver; } } diff --git a/docs/API-SPEC.md b/docs/API-SPEC.md index abb6976..a3069bb 100644 --- a/docs/API-SPEC.md +++ b/docs/API-SPEC.md @@ -261,6 +261,8 @@ If a self-service reset flow is introduced later, it will be added as a new publ } ``` +**M1 implementation notes:** File bytes are written by a `StorageDriver` (default: local filesystem under `STORAGE_ROOT`; production default `/app/data/storage` with a mounted volume per ADR-005). SQLite rows use opaque `storage_path` keys, plus `logical_file_id` and `version` for future metadata-based versioning (not filename suffix rules). Operator checklist: `docs/STORAGE-OPERATIONS.md`. + ### 6.1 POST `/storage/buckets/:bucket/upload` Uploads a file to a bucket scoped to the authenticated user. diff --git a/packages/db/migrations/0007_files_checksum_versioning.sql b/packages/db/migrations/0007_files_checksum_versioning.sql new file mode 100644 index 0000000..0039b7c --- /dev/null +++ b/packages/db/migrations/0007_files_checksum_versioning.sql @@ -0,0 +1,4 @@ +ALTER TABLE `files` ADD `checksum` text;--> statement-breakpoint +ALTER TABLE `files` ADD `logical_file_id` text;--> statement-breakpoint +ALTER TABLE `files` ADD `version` integer DEFAULT 1 NOT NULL;--> statement-breakpoint +UPDATE `files` SET `logical_file_id` = `id` WHERE `logical_file_id` IS NULL; diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index ba655f5..8c5d679 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1773954804206, "tag": "0006_jwks_updated_at_optional", "breakpoints": true + }, + { + "idx": 7, + "version": "6", + "when": 1774000000000, + "tag": "0007_files_checksum_versioning", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/storage.ts b/packages/db/src/schema/storage.ts index 68a0f73..cbbfd11 100644 --- a/packages/db/src/schema/storage.ts +++ b/packages/db/src/schema/storage.ts @@ -3,11 +3,17 @@ import { user } from "./auth"; export const files = sqliteTable("files", { id: text("id").primaryKey(), + /** Same as `id` for M1 uploads; future versioned uploads share the same logical id across rows. */ + logicalFileId: text("logical_file_id").notNull(), + version: integer("version").notNull().default(1), bucket: text("bucket").notNull(), filename: text("filename").notNull(), mimeType: text("mime_type").notNull(), size: integer("size").notNull(), + /** Relative object key passed to the storage driver (opaque, ID-based). */ storagePath: text("storage_path").notNull(), + /** SHA-256 hex of stored bytes (optional for legacy rows). */ + checksum: text("checksum"), ownerId: text("owner_id") .notNull() .references(() => user.id, { onDelete: "cascade" }), diff --git a/packages/storage/package.json b/packages/storage/package.json new file mode 100644 index 0000000..c6073af --- /dev/null +++ b/packages/storage/package.json @@ -0,0 +1,24 @@ +{ + "name": "@appbase/storage", + "version": "0.0.1", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "main": "./src/index.ts", + "types": "./src/index.ts", + "scripts": { + "build": "tsc", + "check-types": "tsc --noEmit", + "dev": "tsc --watch", + "test": "vitest run", + "test:watch": "vitest" + }, + "devDependencies": { + "@appbase/config": "workspace:*", + "@types/node": "^22.13.14", + "typescript": "5.9.2", + "vitest": "^4.1.0" + } +} diff --git a/packages/storage/src/driver.ts b/packages/storage/src/driver.ts new file mode 100644 index 0000000..950ab18 --- /dev/null +++ b/packages/storage/src/driver.ts @@ -0,0 +1,23 @@ +import type { Readable } from "node:stream"; + +export type PutObjectInput = { + /** Opaque relative key (e.g. `objects/`), no user-controlled path segments. */ + objectKey: string; + stream: Readable; +}; + +export type PutObjectResult = { + objectKey: string; + size: number; + checksum: string; +}; + +/** + * Storage backends (local FS, future S3). Callers depend on this interface, not raw `fs`. + */ +export interface StorageDriver { + putObject(input: PutObjectInput): Promise; + getObjectStream(objectKey: string): Readable; + deleteObject(objectKey: string): Promise; + exists(objectKey: string): Promise; +} diff --git a/packages/storage/src/fs-driver.test.ts b/packages/storage/src/fs-driver.test.ts new file mode 100644 index 0000000..392dcfe --- /dev/null +++ b/packages/storage/src/fs-driver.test.ts @@ -0,0 +1,46 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, beforeEach } from "vitest"; +import { Readable } from "node:stream"; +import { FileSystemStorageDriver } from "./fs-driver"; + +describe("FileSystemStorageDriver", () => { + let root: string; + let driver: FileSystemStorageDriver; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "appbase-storage-")); + driver = new FileSystemStorageDriver(root); + }); + + it("putObject, exists, getObjectStream, deleteObject roundtrip", async () => { + const key = "objects/testid001"; + const buf = Buffer.from("hello-bytes"); + await driver.putObject({ objectKey: key, stream: Readable.from(buf) }); + expect(await driver.exists(key)).toBe(true); + + const chunks: Buffer[] = []; + for await (const c of driver.getObjectStream(key)) { + chunks.push(c as Buffer); + } + expect(Buffer.concat(chunks).toString()).toBe("hello-bytes"); + + await driver.deleteObject(key); + expect(await driver.exists(key)).toBe(false); + }); + + it("putObject returns sha256 checksum and size", async () => { + const key = "objects/abc"; + const body = Buffer.from("checksum-me"); + const res = await driver.putObject({ objectKey: key, stream: Readable.from(body) }); + expect(res.size).toBe(body.length); + expect(res.checksum).toMatch(/^[a-f0-9]{64}$/); + }); + + it("rejects object keys that escape root", async () => { + await expect( + driver.putObject({ objectKey: "../escape", stream: Readable.from(Buffer.from("x")) }), + ).rejects.toThrow(/Invalid object key/); + }); +}); diff --git a/packages/storage/src/fs-driver.ts b/packages/storage/src/fs-driver.ts new file mode 100644 index 0000000..b4ccb1a --- /dev/null +++ b/packages/storage/src/fs-driver.ts @@ -0,0 +1,98 @@ +import fs from "node:fs"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { pipeline } from "node:stream/promises"; +import { createReadStream, createWriteStream } from "node:fs"; +import { Transform } from "node:stream"; +import type { Readable } from "node:stream"; +import type { PutObjectInput, PutObjectResult, StorageDriver } from "./driver"; + +function assertSafeObjectKey(root: string, objectKey: string): string { + const abs = path.resolve(root, objectKey); + const rootResolved = path.resolve(root); + if (!abs.startsWith(rootResolved + path.sep) && abs !== rootResolved) { + throw new Error("Invalid object key: path escapes storage root"); + } + return abs; +} + +/** + * Volume-backed filesystem driver: atomic writes (temp + rename), streaming reads, SHA-256 checksum. + */ +export class FileSystemStorageDriver implements StorageDriver { + constructor(private readonly rootDir: string) {} + + async putObject(input: PutObjectInput): Promise { + const targetAbs = assertSafeObjectKey(this.rootDir, input.objectKey); + const tmpAbs = `${targetAbs}.${process.pid}.${Date.now()}.tmp`; + fs.mkdirSync(path.dirname(targetAbs), { recursive: true }); + + const hash = createHash("sha256"); + let size = 0; + const meter = new Transform({ + transform(chunk: Buffer, _enc, cb) { + size += chunk.length; + hash.update(chunk); + cb(null, chunk); + }, + }); + + await pipeline(input.stream, meter, createWriteStream(tmpAbs, { flags: "wx" })); + + try { + fs.renameSync(tmpAbs, targetAbs); + } catch { + try { + fs.unlinkSync(tmpAbs); + } catch { + /* ignore */ + } + throw new Error("Failed to finalize storage object"); + } + + return { + objectKey: input.objectKey, + size, + checksum: hash.digest("hex"), + }; + } + + getObjectStream(objectKey: string): Readable { + const abs = assertSafeObjectKey(this.rootDir, objectKey); + return createReadStream(abs); + } + + async deleteObject(objectKey: string): Promise { + const abs = assertSafeObjectKey(this.rootDir, objectKey); + await fs.promises.unlink(abs).catch((err: NodeJS.ErrnoException) => { + if (err.code === "ENOENT") return; + throw err; + }); + } + + async exists(objectKey: string): Promise { + try { + const abs = assertSafeObjectKey(this.rootDir, objectKey); + await fs.promises.access(abs, fs.constants.R_OK); + return true; + } catch { + return false; + } + } + + /** + * Lists relative object keys under `objects/` for reconciliation (FS driver only). + */ + async listStoredObjectKeys(): Promise { + const objectsDir = path.join(this.rootDir, "objects"); + if (!fs.existsSync(objectsDir)) return []; + const names = await fs.promises.readdir(objectsDir, { withFileTypes: true }); + const out: string[] = []; + for (const ent of names) { + if (ent.isFile()) { + out.push(path.posix.join("objects", ent.name)); + } + } + return out; + } +} diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts new file mode 100644 index 0000000..83353e7 --- /dev/null +++ b/packages/storage/src/index.ts @@ -0,0 +1,3 @@ +export type { StorageDriver, PutObjectInput, PutObjectResult } from "./driver"; +export { FileSystemStorageDriver } from "./fs-driver"; +export { validateBucket, isMimeAllowed } from "./validation"; diff --git a/packages/storage/src/validation.test.ts b/packages/storage/src/validation.test.ts new file mode 100644 index 0000000..51223d9 --- /dev/null +++ b/packages/storage/src/validation.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { validateBucket, isMimeAllowed } from "./validation"; + +describe("validateBucket", () => { + it("accepts valid names", () => { + expect(validateBucket("avatars").valid).toBe(true); + expect(validateBucket("app_assets-1").valid).toBe(true); + }); + + it("rejects empty and traversal-ish names", () => { + expect(validateBucket("").valid).toBe(false); + expect(validateBucket("a/b").valid).toBe(false); + expect(validateBucket("..").valid).toBe(false); + expect(validateBucket("bucket.name").valid).toBe(false); + }); +}); + +describe("isMimeAllowed", () => { + it("allows all when patterns are null", () => { + expect(isMimeAllowed("application/octet-stream", null)).toBe(true); + }); + + it("matches exact and wildcard patterns", () => { + const cfg = "image/png,application/pdf,text/*"; + expect(isMimeAllowed("image/png", cfg)).toBe(true); + expect(isMimeAllowed("text/plain", cfg)).toBe(true); + expect(isMimeAllowed("application/json", cfg)).toBe(false); + }); +}); diff --git a/packages/storage/src/validation.ts b/packages/storage/src/validation.ts new file mode 100644 index 0000000..0e1ab53 --- /dev/null +++ b/packages/storage/src/validation.ts @@ -0,0 +1,38 @@ +const BUCKET_MAX = 64; +const BUCKET_REGEX = /^[a-zA-Z0-9_-]+$/; + +export function validateBucket(name: string): { valid: true } | { valid: false; error: string } { + if (typeof name !== "string" || name.length === 0) { + return { valid: false, error: "Bucket name is required" }; + } + if (name.length > BUCKET_MAX) { + return { valid: false, error: `Bucket name must be at most ${BUCKET_MAX} characters` }; + } + if (!BUCKET_REGEX.test(name)) { + return { + valid: false, + error: "Bucket name may only contain letters, numbers, hyphens, and underscores", + }; + } + return { valid: true }; +} + +/** + * MIME allowlist (e.g. from env `STORAGE_ALLOWED_MIME`). `configuredPatterns === null` → allow all. + */ +export function isMimeAllowed(mimeType: string, configuredPatterns: string | null): boolean { + if (!configuredPatterns) return true; + const normalized = mimeType.trim().toLowerCase(); + const patterns = configuredPatterns + .split(",") + .map((s) => s.trim().toLowerCase()) + .filter(Boolean); + for (const p of patterns) { + if (p === normalized) return true; + if (p.endsWith("/*")) { + const prefix = p.slice(0, -1); + if (normalized.startsWith(prefix)) return true; + } + } + return false; +} diff --git a/packages/storage/tsconfig.json b/packages/storage/tsconfig.json new file mode 100644 index 0000000..e0db756 --- /dev/null +++ b/packages/storage/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../config/tsconfig.base.json", + "compilerOptions": { + "module": "ESNext", + "moduleResolution": "Bundler", + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/packages/storage/vitest.config.ts b/packages/storage/vitest.config.ts new file mode 100644 index 0000000..f624398 --- /dev/null +++ b/packages/storage/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6280f3c..1953e6d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: '@appbase/db': specifier: workspace:* version: link:../../packages/db + '@appbase/storage': + specifier: workspace:* + version: link:../../packages/storage '@appbase/types': specifier: workspace:* version: link:../../packages/types @@ -242,6 +245,21 @@ importers: specifier: ^3.0.9 version: 3.2.4(@types/node@22.15.3)(jiti@2.6.1)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.2) + packages/storage: + devDependencies: + '@appbase/config': + specifier: workspace:* + version: link:../config + '@types/node': + specifier: ^22.13.14 + version: 22.15.3 + typescript: + specifier: 5.9.2 + version: 5.9.2 + vitest: + specifier: ^4.1.0 + version: 4.1.0(@types/node@22.15.3)(vite@7.3.1(@types/node@22.15.3)(jiti@2.6.1)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.2)) + packages/types: devDependencies: '@appbase/config': @@ -3426,6 +3444,7 @@ packages: typescript@5.9.2: resolution: {integrity: sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==} engines: {node: '>=14.17'} + hasBin: true unbox-primitive@1.1.0: resolution: {integrity: sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==} From 4dce65b731677dd29c7d4090cf6ddc4d2c7aded1 Mon Sep 17 00:00:00 2001 From: Nabil Mouzouna Date: Wed, 25 Mar 2026 17:18:28 +0100 Subject: [PATCH 02/11] FEAT: creating storage service inside the SDK --- .../routes/storage.sdk-integration.test.ts | 147 ++++++++++ apps/api/src/routes/storage.ts | 2 + docs/API-SPEC.md | 2 +- docs/adr/ADR-005-file-storage-strategy.md | 2 +- packages/sdk/README.md | 2 +- packages/sdk/docs/storage-service.md | 134 +++++++++ packages/sdk/src/auth.ts | 21 ++ packages/sdk/src/storage.test.ts | 188 +++++++++++++ packages/sdk/src/storage.ts | 259 +++++++++++++++--- 9 files changed, 712 insertions(+), 45 deletions(-) create mode 100644 apps/api/src/routes/storage.sdk-integration.test.ts create mode 100644 packages/sdk/docs/storage-service.md create mode 100644 packages/sdk/src/storage.test.ts diff --git a/apps/api/src/routes/storage.sdk-integration.test.ts b/apps/api/src/routes/storage.sdk-integration.test.ts new file mode 100644 index 0000000..646cf7b --- /dev/null +++ b/apps/api/src/routes/storage.sdk-integration.test.ts @@ -0,0 +1,147 @@ +/** + * Integration tests: SDK storage client against live API (API-SPEC §6). + */ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, beforeAll, afterAll, beforeEach } from "vitest"; +import { AppBase } from "@appbase/sdk"; +import { buildApp } from "../app"; +import { loadEnv } from "../config/env"; + +describe("SDK storage integration", () => { + const port = 38600 + (Math.floor(Math.random() * 500) % 500); + const baseUrl = `http://127.0.0.1:${port}`; + + let tmpDir: string; + let testEnv: ReturnType; + let app: Awaited>; + let appbase: AppBase; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "appbase-sdk-stor-")); + testEnv = loadEnv({ + ...process.env, + NODE_ENV: "test", + DB_PATH: path.join(tmpDir, "app.sqlite"), + STORAGE_ROOT: path.join(tmpDir, "storage"), + AUTH_SECRET: "test-secret-must-be-at-least-32-characters-long", + BASE_URL: baseUrl, + STORAGE_ALLOWED_MIME: "*", + }); + app = await buildApp({ env: testEnv }); + await app.listen({ port, host: "127.0.0.1" }); + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + const email = `sdk-st-${Date.now()}@example.com`; + const password = "SecurePassword123!"; + + await app.inject({ + method: "POST", + url: "/auth/register", + payload: { email, password }, + }); + + appbase = AppBase.init({ + endpoint: baseUrl, + apiKey: "hs_live_test_key", + }); + + await appbase.auth.signIn({ email, password }); + }); + + it("upload, list, download, remove happy path", async () => { + const up = await appbase.storage.upload( + "docs", + new Blob(["hello-sdk"], { type: "text/plain" }), + { filename: "note.txt" }, + ); + + expect(up.file.id).toBeDefined(); + expect(up.file.bucket).toBe("docs"); + expect(up.url).toContain(up.file.id); + + const listed = await appbase.storage.list("docs"); + expect(listed.total).toBeGreaterThanOrEqual(1); + expect(listed.files.some((f) => f.id === up.file.id)).toBe(true); + + const buf = await appbase.storage.download("docs", up.file.id, { as: "buffer" }); + expect(buf.toString()).toBe("hello-sdk"); + + await appbase.storage.remove("docs", up.file.id); + + const listed2 = await appbase.storage.list("docs"); + expect(listed2.files.every((f) => f.id !== up.file.id)).toBe(true); + }); + + it("SDK validates bucket before request", async () => { + await expect(appbase.storage.upload("bad/name", new Blob(["x"]))).rejects.toMatchObject({ + name: "StorageError", + code: "VALIDATION_ERROR", + }); + }); + + it("throws when session cleared before storage call", async () => { + await appbase.auth.signOut(); + await expect(appbase.storage.list("docs")).rejects.toThrow(/Not authenticated|signIn/i); + }); + + it("ownership: other user gets NOT_FOUND on download", async () => { + const up = await appbase.storage.upload("private", new Blob(["secret"]), { filename: "a.bin" }); + + const email2 = `sdk-st-2-${Date.now()}@example.com`; + await app.inject({ + method: "POST", + url: "/auth/register", + payload: { email: email2, password: "SecurePassword123!" }, + }); + + const other = AppBase.init({ endpoint: baseUrl, apiKey: "hs_live_test_key" }); + await other.auth.signIn({ email: email2, password: "SecurePassword123!" }); + + await expect(other.storage.download("private", up.file.id, { as: "buffer" })).rejects.toMatchObject({ + name: "StorageError", + code: "NOT_FOUND", + }); + }); + + it("disallowed MIME propagated", async () => { + const mimePort = port + 6000; + const mimeBase = `http://127.0.0.1:${mimePort}`; + const envMime = loadEnv({ + ...process.env, + NODE_ENV: "test", + DB_PATH: path.join(tmpDir, `mime-${Date.now()}.sqlite`), + STORAGE_ROOT: path.join(tmpDir, `mime-st-${Date.now()}`), + AUTH_SECRET: "test-secret-must-be-at-least-32-characters-long", + BASE_URL: mimeBase, + STORAGE_ALLOWED_MIME: "image/png", + }); + const appMime = await buildApp({ env: envMime }); + await appMime.listen({ port: mimePort, host: "127.0.0.1" }); + + const email = `sdk-st-mime-${Date.now()}@example.com`; + await appMime.inject({ + method: "POST", + url: "/auth/register", + payload: { email, password: "SecurePassword123!" }, + }); + + const client = AppBase.init({ endpoint: mimeBase, apiKey: "hs_live_test_key" }); + await client.auth.signIn({ email, password: "SecurePassword123!" }); + + await expect( + client.storage.upload("b", new Blob(["x"], { type: "text/plain" }), { filename: "x.txt" }), + ).rejects.toMatchObject({ + name: "StorageError", + code: "VALIDATION_ERROR", + }); + + await appMime.close(); + }); +}); diff --git a/apps/api/src/routes/storage.ts b/apps/api/src/routes/storage.ts index 5d51d73..2973fc4 100644 --- a/apps/api/src/routes/storage.ts +++ b/apps/api/src/routes/storage.ts @@ -330,6 +330,8 @@ export async function registerStorageRoutes(app: FastifyInstance) { } const row = rows[0]!; + // M1: not transactional across DB + FS. Row removed first so the API never leaves metadata + // pointing at missing objects; failed disk delete → orphan bytes (reconcile + manual GC). await app.db.delete(files).where(and(eq(files.id, fileId), eq(files.ownerId, userId))); await driver.deleteObject(row.storagePath).catch((err) => { request.log.warn({ err, storagePath: row.storagePath }, "storage.delete.object_cleanup_failed"); diff --git a/docs/API-SPEC.md b/docs/API-SPEC.md index a3069bb..027e47f 100644 --- a/docs/API-SPEC.md +++ b/docs/API-SPEC.md @@ -261,7 +261,7 @@ If a self-service reset flow is introduced later, it will be added as a new publ } ``` -**M1 implementation notes:** File bytes are written by a `StorageDriver` (default: local filesystem under `STORAGE_ROOT`; production default `/app/data/storage` with a mounted volume per ADR-005). SQLite rows use opaque `storage_path` keys, plus `logical_file_id` and `version` for future metadata-based versioning (not filename suffix rules). Operator checklist: `docs/STORAGE-OPERATIONS.md`. +**M1 implementation notes:** File bytes are written by a `StorageDriver` (default: local filesystem under `STORAGE_ROOT`; production default `/app/data/storage` with a mounted volume per ADR-005). SQLite rows use opaque `storage_path` keys, plus `logical_file_id` and `version` for future metadata-based versioning (not filename suffix rules). **Deletes** commit metadata removal before best-effort blob removal, so orphan objects on disk are possible if cleanup fails; use reconciliation + manual remediation as documented in `docs/STORAGE-OPERATIONS.md`. Operator checklist: same file. ### 6.1 POST `/storage/buckets/:bucket/upload` diff --git a/docs/adr/ADR-005-file-storage-strategy.md b/docs/adr/ADR-005-file-storage-strategy.md index f283c62..50ae261 100644 --- a/docs/adr/ADR-005-file-storage-strategy.md +++ b/docs/adr/ADR-005-file-storage-strategy.md @@ -148,7 +148,7 @@ Use **metadata tracking** as the source of truth, not filename suffix parsing. - Write uploads atomically (temp file + rename). - Store checksum/hash in metadata where feasible. - Enforce max upload size and content-type allowlist policy. -- Ensure delete path removes file bytes and metadata coherently. +- Delete path: M1 removes the **metadata row first**, then best-effort object delete. This is **not** a cross-store transaction; failed object removal can leave **orphan bytes** until an operator runs reconciliation and removes files manually (see `docs/STORAGE-OPERATIONS.md`). - Add periodic reconciliation tooling (detect metadata without object and object without metadata). --- diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 354a737..15fca86 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -26,4 +26,4 @@ const { items } = await todos.list(); |---------|--------| | **`appbase.auth`** | [Auth docs](./docs/auth-service.md) | | **`appbase.db`** | [Database docs](./docs/db-service.md) | -| **`appbase.storage`** | Placeholder | +| **`appbase.storage`** | [Storage docs](./docs/storage-service.md) | diff --git a/packages/sdk/docs/storage-service.md b/packages/sdk/docs/storage-service.md new file mode 100644 index 0000000..27ba28c --- /dev/null +++ b/packages/sdk/docs/storage-service.md @@ -0,0 +1,134 @@ +# Storage service (`appbase.storage`) + +Typed client for **API-SPEC §6** (`docs/API-SPEC.md` in the monorepo) — upload, list, download, and delete user-scoped files in buckets. + +## Auth expectations + +- Every call uses **`x-api-key`** (from `AppBase.init`) and **`Authorization: Bearer `** only. +- **`auth.signIn` / `auth.signUp`** must run first so the SDK holds an access JWT (and browser session cookie for `/auth/refresh`). +- Before each storage request the SDK calls **`auth.ensureAccessToken()`**, which refreshes the JWT via **`POST /auth/refresh`** when it is near expiry (cookie-based). Storage requests themselves do **not** send refresh tokens in `Authorization`. + +## Quick start + +```ts +import { AppBase } from "@appbase/sdk"; + +const appbase = AppBase.init({ + endpoint: "http://localhost:3000", + apiKey: process.env.APPBASE_API_KEY!, + sessionStorageKey: "my_app", // browser: persist access slice + enable refresh +}); + +await appbase.auth.signIn({ email: "u@example.com", password: "…" }); + +const { file, url } = await appbase.storage.upload("avatars", fileInput, { + filename: "profile.png", +}); +``` + +## API + +| Method | Description | +|--------|-------------| +| `upload(bucket, input, options?)` | `POST /storage/buckets/:bucket/upload`, multipart field **`file`**. | +| `list(bucket)` | `GET /storage/buckets/:bucket` → `{ files, total }`. | +| `download(bucket, fileId, options?)` | Binary `GET`; default **`Blob`** in browser, **`Buffer`** in Node (override with `{ as: "blob" \| "buffer" }`). | +| `remove(bucket, fileId)` | `DELETE ...`; alias **`delete`**. | +| `getUrl(bucket, fileId)` | Builds public path string (no I/O). | + +### Upload inputs + +| Environment | Accepted types | +|-------------|----------------| +| Browser | `File`, `Blob`, `FormData` (must include field **`file`**) | +| Node | `Buffer`, `Uint8Array`, **`Readable`** stream *(read fully into memory before send — avoid huge files in M1)* | + +Options: `{ filename?, contentType? }` (ignored for `File` where `File.name` is used). + +### Download outputs + +- Default: `Blob` when `window` exists, else `Buffer`. +- Full response bodies are buffered in memory — large files should be handled carefully until streaming is added server/SDK-side. + +### Types + +- **`StorageFile`** — alias of `FileRecord`: `id`, `bucket`, `filename`, `mimeType`, `size`, `ownerId`, `createdAt`. +- **`StorageError`** — `name === "StorageError"`, `code` (e.g. `VALIDATION_ERROR`, `INVALID_TOKEN`, `NOT_FOUND`, `PAYLOAD_TOO_LARGE`), `status` (HTTP when available), `message`. + +Checksum / version fields are **not** exposed until the public API contract includes them. + +## Examples + +### Browser — file input + +```ts +const input = document.querySelector("input[type=file]") as HTMLInputElement; +const file = input.files?.[0]; +if (!file) return; + +const { file: meta, url } = await appbase.storage.upload("uploads", file); +console.log(meta.id, url); +``` + +### Browser — Blob + +```ts +await appbase.storage.upload("docs", new Blob(["hello"], { type: "text/plain" }), { + filename: "note.txt", +}); +``` + +### Node — Buffer + +```ts +import { AppBase } from "@appbase/sdk"; +import fs from "node:fs"; + +const appbase = AppBase.init({ + endpoint: "http://127.0.0.1:3000", + apiKey: process.env.APPBASE_API_KEY!, +}); + +await appbase.auth.signIn({ email: process.env.USER_EMAIL!, password: process.env.USER_PASSWORD! }); + +const buf = fs.readFileSync("./report.pdf"); +await appbase.storage.upload("reports", buf, { + filename: "report.pdf", + contentType: "application/pdf", +}); + +const out = await appbase.storage.download("reports", fileId, { as: "buffer" }); +fs.writeFileSync("./out.pdf", out); +``` + +### Node — stream (fully buffered) + +```ts +import { createReadStream } from "node:fs"; + +await appbase.storage.upload("raw", createReadStream("./big.bin"), { + filename: "big.bin", + contentType: "application/octet-stream", +}); +``` + +## Error handling + +```ts +import { AppBase, StorageError } from "@appbase/sdk"; + +try { + await appbase.storage.download("b", id, { as: "blob" }); +} catch (e) { + if (e instanceof StorageError) { + console.error(e.code, e.message, e.status); + } + throw e; +} +``` + +Client-side checks (empty bucket, invalid `fileId` pattern) throw **`StorageError`** with code **`VALIDATION_ERROR`** before any network call. + +## Server-side storage (ADR-005) + +File bytes are stored by a **`StorageDriver`** on the server (default: filesystem under `STORAGE_ROOT`), not in the SDK. Deletes are not a single DB+FS transaction in M1; operational notes live in [`docs/STORAGE-OPERATIONS.md`](../../../docs/STORAGE-OPERATIONS.md) and [ADR-005](../../../docs/adr/ADR-005-file-storage-strategy.md). diff --git a/packages/sdk/src/auth.ts b/packages/sdk/src/auth.ts index e73e8b0..366cbb4 100644 --- a/packages/sdk/src/auth.ts +++ b/packages/sdk/src/auth.ts @@ -253,4 +253,25 @@ export class AuthClient { getAccessToken(): string | null { return this.session?.accessToken ?? null; } + + /** + * Ensures a usable access JWT for protected API routes (`/storage/*`, `/db/*`). + * Runs the startup restore hook, then refreshes via session cookie when the token is near expiry. + */ + ensureAccessToken = async (): Promise => { + await this.initPromise; + if (!this.session) { + throw new Error( + "Not authenticated. Call auth.signIn or auth.signUp before storage or database operations.", + ); + } + if (this.isAccessTokenStale()) { + await this.refreshAccessToken(); + } + const token = this.session.accessToken; + if (!token) { + throw new Error("No access token available."); + } + return token; + }; } diff --git a/packages/sdk/src/storage.test.ts b/packages/sdk/src/storage.test.ts new file mode 100644 index 0000000..e1b163a --- /dev/null +++ b/packages/sdk/src/storage.test.ts @@ -0,0 +1,188 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; +import { StorageClient } from "./storage"; +import type { AuthClient } from "./auth"; + +const mockConfig = { + endpoint: "http://api.test", + apiKey: "hs_live_test", +}; + +function mockAuth(token = "mock-token"): AuthClient & { ensureAccessToken: ReturnType } { + return { + getAccessToken: () => token, + ensureAccessToken: vi.fn().mockResolvedValue(token), + } as AuthClient & { ensureAccessToken: ReturnType }; +} + +describe("StorageClient", () => { + let fetchMock: ReturnType; + + beforeEach(() => { + fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("upload sends multipart field file and returns UploadResponse", async () => { + fetchMock.mockResolvedValue({ + ok: true, + json: async () => ({ + data: { + file: { + id: "f1", + bucket: "docs", + filename: "a.txt", + mimeType: "text/plain", + size: 4, + ownerId: "u1", + createdAt: "2026-01-01T00:00:00.000Z", + }, + url: "/storage/buckets/docs/f1", + }, + }), + }); + + const auth = mockAuth(); + const storage = new StorageClient(mockConfig, auth); + const file = new Blob(["hey"], { type: "text/plain" }); + const result = await storage.upload("docs", file, { filename: "a.txt" }); + + expect(auth.ensureAccessToken).toHaveBeenCalled(); + expect(result.url).toContain("/storage/buckets/docs/f1"); + expect(result.file.id).toBe("f1"); + const call = fetchMock.mock.calls[0]!; + expect(call[0]).toBe("http://api.test/storage/buckets/docs/upload"); + const init = call[1] as RequestInit; + expect(init.method).toBe("POST"); + expect(init.credentials).toBe("include"); + const headers = init.headers as Headers; + expect(headers.get("x-api-key")).toBe("hs_live_test"); + expect(headers.get("Authorization")).toBe("Bearer mock-token"); + expect(headers.get("Content-Type")).toBeNull(); + expect(init.body).toBeInstanceOf(FormData); + }); + + it("rejects empty bucket before fetch", async () => { + const storage = new StorageClient(mockConfig, mockAuth()); + await expect(storage.upload("", new Blob([]))).rejects.toMatchObject({ + name: "StorageError", + code: "VALIDATION_ERROR", + }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("maps 413 payload errors to StorageError", async () => { + fetchMock.mockResolvedValue({ + ok: false, + status: 413, + text: async () => + JSON.stringify({ + success: false, + error: { code: "PAYLOAD_TOO_LARGE", message: "Too big" }, + }), + }); + + const storage = new StorageClient(mockConfig, mockAuth()); + await expect(storage.upload("docs", new Blob(["x"]))).rejects.toMatchObject({ + name: "StorageError", + code: "PAYLOAD_TOO_LARGE", + status: 413, + }); + }); + + it("maps API error envelope to StorageError", async () => { + fetchMock.mockResolvedValue({ + ok: false, + status: 404, + text: async () => + JSON.stringify({ + success: false, + error: { code: "NOT_FOUND", message: "File not found" }, + }), + }); + + const storage = new StorageClient(mockConfig, mockAuth()); + await expect(storage.list("docs")).rejects.toMatchObject({ + name: "StorageError", + code: "NOT_FOUND", + message: "File not found", + status: 404, + }); + }); + + it("list returns files and total", async () => { + fetchMock.mockResolvedValue({ + ok: true, + json: async () => ({ + data: { + total: 1, + files: [ + { + id: "f1", + bucket: "b", + filename: "x.png", + mimeType: "image/png", + size: 10, + ownerId: "u1", + createdAt: "2026-01-01T00:00:00.000Z", + }, + ], + }, + }), + }); + + const storage = new StorageClient(mockConfig, mockAuth()); + const r = await storage.list("b"); + expect(r.total).toBe(1); + expect(r.files[0]!.filename).toBe("x.png"); + }); + + it("download with as buffer uses arrayBuffer", async () => { + fetchMock.mockResolvedValue({ + ok: true, + arrayBuffer: async () => new Uint8Array([1, 2, 3]).buffer, + }); + + const storage = new StorageClient(mockConfig, mockAuth()); + const buf = await storage.download("docs", "abcdefgh12345678901", { as: "buffer" }); + expect(Buffer.isBuffer(buf)).toBe(true); + expect([...(buf as Buffer)]).toEqual([1, 2, 3]); + }); + + it("download with as blob calls res.blob", async () => { + const blob = new Blob([new Uint8Array([9])]); + fetchMock.mockResolvedValue({ + ok: true, + blob: async () => blob, + }); + + const storage = new StorageClient(mockConfig, mockAuth()); + const out = await storage.download("docs", "abcdefgh12345678901", { as: "blob" }); + expect(out).toBe(blob); + }); + + it("remove calls DELETE", async () => { + fetchMock.mockResolvedValue({ + ok: true, + json: async () => ({ success: true, data: { deleted: true } }), + }); + + const storage = new StorageClient(mockConfig, mockAuth()); + await storage.remove("docs", "abcdefgh12345678901"); + + expect(fetchMock).toHaveBeenCalledWith( + "http://api.test/storage/buckets/docs/abcdefgh12345678901", + expect.objectContaining({ method: "DELETE" }), + ); + }); + + it("getUrl encodes path segments", () => { + const storage = new StorageClient(mockConfig, mockAuth()); + expect(storage.getUrl("my-bucket", "abcd1234abcd1234abcd12")).toBe( + "http://api.test/storage/buckets/my-bucket/abcd1234abcd1234abcd12", + ); + }); +}); diff --git a/packages/sdk/src/storage.ts b/packages/sdk/src/storage.ts index 4d33dc3..017ebd6 100644 --- a/packages/sdk/src/storage.ts +++ b/packages/sdk/src/storage.ts @@ -2,72 +2,247 @@ import type { AppBaseConfig } from "./appbase"; import type { AuthClient } from "./auth"; import type { FileRecord, BucketListResponse, UploadResponse } from "@appbase/types"; +/** File metadata row returned by the API (API-SPEC §6). */ +export type StorageFile = FileRecord; + +export class StorageError extends Error { + constructor( + message: string, + public readonly code: string, + public readonly status?: number, + ) { + super(message); + this.name = "StorageError"; + } +} + +export type StorageUploadOptions = { + /** Original filename (Blob/Buffer/stream). Ignored when `input` is a `File` (uses `File.name`). */ + filename?: string; + /** MIME type hint for Blob/Buffer/stream bodies. */ + contentType?: string; +}; + +/** Supported upload bodies: browser `File` / `Blob` / `FormData` (`file` field); Node `Buffer` / `Uint8Array` / readable stream. */ +export type StorageUploadInput = + | File + | Blob + | FormData + | Buffer + | Uint8Array + | import("node:stream").Readable; + +export type StorageDownloadOptions = { + /** + * Return shape. Default: `blob` in browsers, `buffer` in Node. + * Streams are fully buffered; avoid multi‑GB files until a streaming API exists. + */ + as?: "blob" | "buffer"; +}; + +const BUCKET_PATTERN = /^[a-zA-Z0-9_-]{1,64}$/; + +function parseApiError(text: string, status: number): { code: string; message: string } { + try { + const j = JSON.parse(text) as { error?: { code?: string; message?: string } }; + if (j.error?.message) { + return { + code: j.error.code ?? "REQUEST_FAILED", + message: j.error.message, + }; + } + } catch { + /* ignore */ + } + return { code: status === 401 ? "INVALID_TOKEN" : "HTTP_ERROR", message: text || `HTTP ${status}` }; +} + +function assertValidBucket(bucket: string): void { + if (typeof bucket !== "string" || !bucket.trim()) { + throw new StorageError("Bucket name is required", "VALIDATION_ERROR"); + } + if (!BUCKET_PATTERN.test(bucket)) { + throw new StorageError( + "Bucket must be 1–64 characters: letters, digits, hyphens, and underscores only", + "VALIDATION_ERROR", + ); + } +} + +function assertValidFileId(fileId: string): void { + if (typeof fileId !== "string" || !fileId.trim()) { + throw new StorageError("fileId is required", "VALIDATION_ERROR"); + } + if (!/^[a-zA-Z0-9_-]{8,64}$/.test(fileId)) { + throw new StorageError("fileId format is invalid", "VALIDATION_ERROR"); + } +} + +function isFile(x: unknown): x is File { + return typeof File !== "undefined" && x instanceof File; +} + +function uint8ToBlobSlice(u8: Uint8Array, contentType?: string): Blob { + const copy = Uint8Array.from(u8); + return new Blob([copy], contentType ? { type: contentType } : undefined); +} + +async function buildUploadFormData( + input: StorageUploadInput, + options?: StorageUploadOptions, +): Promise { + if (input instanceof FormData) { + if (!input.has("file")) { + throw new StorageError("FormData must include a multipart field named \"file\"", "VALIDATION_ERROR"); + } + return input; + } + + const form = new FormData(); + const filename = options?.filename; + const contentType = options?.contentType; + + if (isFile(input)) { + form.append("file", input); + return form; + } + + if (input instanceof Blob) { + form.append("file", input, filename ?? "upload"); + return form; + } + + if (typeof Buffer !== "undefined" && Buffer.isBuffer(input)) { + const u8 = new Uint8Array(input.buffer, input.byteOffset, input.byteLength); + const blob = uint8ToBlobSlice(u8, contentType); + form.append("file", blob, filename ?? "upload"); + return form; + } + + if (input instanceof Uint8Array) { + const blob = uint8ToBlobSlice(input, contentType); + form.append("file", blob, filename ?? "upload"); + return form; + } + + const { Readable } = await import("node:stream"); + const { blob: streamToBlob } = await import("node:stream/consumers"); + if (Readable.isReadable?.(input as NodeJS.ReadableStream)) { + const b = await streamToBlob(input as import("node:stream").Readable); + form.append("file", b as unknown as Blob, filename ?? "upload"); + return form; + } + + throw new StorageError("Unsupported upload body type", "VALIDATION_ERROR"); +} + export class StorageClient { constructor( private config: AppBaseConfig, - private auth: AuthClient + private auth: AuthClient, ) {} private get baseUrl() { return `${this.config.endpoint}/storage`; } - private headers(): Record { - const token = this.auth.getAccessToken(); + private async authorizedInit(base: RequestInit): Promise { + const token = await this.auth.ensureAccessToken(); + const headers = new Headers(base.headers); + headers.set("x-api-key", this.config.apiKey); + headers.set("Authorization", `Bearer ${token}`); + if (base.body instanceof FormData) { + headers.delete("Content-Type"); + } return { - "x-api-key": this.config.apiKey, - ...(token ? { Authorization: `Bearer ${token}` } : {}), + ...base, + headers, + credentials: "include", }; } - async upload(bucket: string, file: File | Blob, filename?: string): Promise { - const form = new FormData(); - form.append("file", file, filename ?? (file instanceof File ? file.name : "upload")); - const res = await fetch(`${this.baseUrl}/buckets/${bucket}/upload`, { + private async handleResponse(res: Response): Promise { + if (res.ok) return; + const text = await res.text(); + const { code, message } = parseApiError(text, res.status); + throw new StorageError(message, code, res.status); + } + + /** + * Public URL path for a file (same host as `endpoint`). Does not verify existence. + */ + getUrl(bucket: string, fileId: string): string { + assertValidBucket(bucket); + assertValidFileId(fileId); + return `${this.baseUrl}/buckets/${encodeURIComponent(bucket)}/${encodeURIComponent(fileId)}`; + } + + /** + * Upload a file (`multipart` field name `file` per API-SPEC §6). + */ + async upload( + bucket: string, + input: StorageUploadInput, + options?: StorageUploadOptions, + ): Promise { + assertValidBucket(bucket); + const form = await buildUploadFormData(input, options); + const init = await this.authorizedInit({ method: "POST", - headers: this.headers(), body: form, }); - if (!res.ok) throw new Error(await res.text()); - const json = await res.json() as { data: UploadResponse }; + const res = await fetch(`${this.baseUrl}/buckets/${encodeURIComponent(bucket)}/upload`, init); + await this.handleResponse(res); + const json = (await res.json()) as { data: UploadResponse }; return json.data; } - async getUrl(bucket: string, fileId: string): Promise { - return `${this.baseUrl}/buckets/${bucket}/${fileId}`; + async list(bucket: string): Promise { + assertValidBucket(bucket); + const init = await this.authorizedInit({ method: "GET" }); + const res = await fetch(`${this.baseUrl}/buckets/${encodeURIComponent(bucket)}`, init); + await this.handleResponse(res); + const json = (await res.json()) as { data: BucketListResponse }; + return json.data; } - async download(bucket: string, fileId: string): Promise { - const res = await fetch(`${this.baseUrl}/buckets/${bucket}/${fileId}`, { - headers: this.headers(), - }); - if (!res.ok) throw new Error(await res.text()); - return res.blob(); - } + /** + * Download file bytes. Default: `Blob` in browser, `Buffer` in Node. + */ + async download( + bucket: string, + fileId: string, + options?: StorageDownloadOptions, + ): Promise { + assertValidBucket(bucket); + assertValidFileId(fileId); + const init = await this.authorizedInit({ method: "GET" }); + const res = await fetch( + `${this.baseUrl}/buckets/${encodeURIComponent(bucket)}/${encodeURIComponent(fileId)}`, + init, + ); + await this.handleResponse(res); - async delete(bucket: string, fileId: string): Promise { - const res = await fetch(`${this.baseUrl}/buckets/${bucket}/${fileId}`, { - method: "DELETE", - headers: this.headers(), - }); - if (!res.ok) throw new Error(await res.text()); - } + const defaultAs: StorageDownloadOptions["as"] = + options?.as ?? (typeof globalThis.window !== "undefined" ? "blob" : "buffer"); - async list(bucket: string): Promise { - const res = await fetch(`${this.baseUrl}/buckets/${bucket}`, { - headers: this.headers(), - }); - if (!res.ok) throw new Error(await res.text()); - const json = await res.json() as { data: BucketListResponse }; - return json.data; + if (defaultAs === "buffer") { + return Buffer.from(await res.arrayBuffer()); + } + return res.blob(); } - async getFile(bucket: string, fileId: string): Promise { - const res = await fetch(`${this.baseUrl}/buckets/${bucket}/${fileId}/meta`, { - headers: this.headers(), - }); - if (!res.ok) throw new Error(await res.text()); - const json = await res.json() as { data: FileRecord }; - return json.data; + async remove(bucket: string, fileId: string): Promise { + assertValidBucket(bucket); + assertValidFileId(fileId); + const init = await this.authorizedInit({ method: "DELETE" }); + const res = await fetch( + `${this.baseUrl}/buckets/${encodeURIComponent(bucket)}/${encodeURIComponent(fileId)}`, + init, + ); + await this.handleResponse(res); } + + /** Alias for {@link remove}. */ + delete = this.remove; } From a82b62cfd78c9a8e926343fd4af4b5eafa37a57d Mon Sep 17 00:00:00 2001 From: Nabil Mouzouna Date: Wed, 25 Mar 2026 18:35:09 +0100 Subject: [PATCH 03/11] FIX: fixing path for storage to store under api/data/storage/objects + fixing required api key in local dev --- apps/example/app/dashboard/page.tsx | 114 ++++++++- apps/example/app/profile/page.tsx | 6 + apps/example/app/sign-in/page.tsx | 37 ++- apps/example/app/sign-up/page.tsx | 20 +- apps/example/components/ProfileModal.tsx | 288 +++++++++++++++++++++++ apps/example/components/ui/Modal.tsx | 6 +- apps/example/lib/api-errors.ts | 10 + apps/example/lib/profile-schema.ts | 21 ++ 8 files changed, 492 insertions(+), 10 deletions(-) create mode 100644 apps/example/app/profile/page.tsx create mode 100644 apps/example/components/ProfileModal.tsx create mode 100644 apps/example/lib/api-errors.ts create mode 100644 apps/example/lib/profile-schema.ts diff --git a/apps/example/app/dashboard/page.tsx b/apps/example/app/dashboard/page.tsx index b158f0d..dbeb22a 100644 --- a/apps/example/app/dashboard/page.tsx +++ b/apps/example/app/dashboard/page.tsx @@ -1,12 +1,20 @@ "use client"; import Link from "next/link"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useRouter } from "next/navigation"; import { z } from "zod"; import { useAppBase, useAuth, useRequireAuth } from "@/lib/appbase"; +import { shouldClearSessionForApiError } from "@/lib/api-errors"; import type { DbRecord } from "@appbase/sdk"; import { Button, Input, Textarea, Modal, Badge, CardHeader } from "@/components/ui"; +import { ProfileModal } from "@/components/ProfileModal"; +import { + AVATAR_BUCKET, + ProfileRowSchema, + defaultDisplayName, + type ProfileData, +} from "@/lib/profile-schema"; const TodoSchema = z.object({ title: z.string(), @@ -22,12 +30,30 @@ type Todo = DbRecord; export default function DashboardPage() { const router = useRouter(); const appBase = useAppBase(); - const { signOut } = useAuth(); + const { signOut, getCurrentUser } = useAuth(); + + const redirectToSignInAfterFatalApiError = useCallback(async () => { + try { + await signOut(); + } catch { + /* still navigate */ + } + router.replace("/sign-in?reason=config"); + }, [signOut, router]); const { authState, authenticated, user } = useRequireAuth("/sign-in", router); const todosCollection = useMemo( () => appBase.db.collection("todos", TodoSchema), [appBase], ); + const profilesCollection = useMemo( + () => appBase.db.collection("profiles", ProfileRowSchema), + [appBase], + ); + + const navAvatarUrlRef = useRef(null); + const [navAvatarUrl, setNavAvatarUrl] = useState(null); + const [navAvatarLetter, setNavAvatarLetter] = useState("?"); + const [profileModalOpen, setProfileModalOpen] = useState(false); const [todos, setTodos] = useState([]); const [title, setTitle] = useState(""); @@ -56,10 +82,14 @@ export default function DashboardPage() { }); setTodos(res.items); } catch (err) { + if (shouldClearSessionForApiError(err)) { + await redirectToSignInAfterFatalApiError(); + return; + } const msg = err instanceof Error ? err.message : "Failed to load todos"; setError(msg); } - }, [todosCollection, filter]); + }, [todosCollection, filter, redirectToSignInAfterFatalApiError]); useEffect(() => { if (authState === null || !authenticated) return; @@ -67,6 +97,62 @@ export default function DashboardPage() { void loadTodos(); }, [authState, authenticated, loadTodos, user?.id]); + const setNavAvatarUrlSafe = useCallback((next: string | null) => { + if (navAvatarUrlRef.current) { + URL.revokeObjectURL(navAvatarUrlRef.current); + navAvatarUrlRef.current = null; + } + if (next) navAvatarUrlRef.current = next; + setNavAvatarUrl(next); + }, []); + + const refreshNavAvatar = useCallback(async () => { + if (!user?.email) return; + try { + const { items } = await profilesCollection.list({ limit: 1 }); + const row = items[0]; + const u = getCurrentUser(); + const label = + row?.data.displayName ?? defaultDisplayName(user.email, u?.customIdentity ?? undefined); + setNavAvatarLetter(label.slice(0, 1).toUpperCase()); + const fid = row?.data.avatarFileId; + if (!fid) { + setNavAvatarUrlSafe(null); + return; + } + const blob = (await appBase.storage.download(AVATAR_BUCKET, fid, { as: "blob" })) as Blob; + setNavAvatarUrlSafe(URL.createObjectURL(blob)); + } catch (err) { + if (shouldClearSessionForApiError(err)) { + await redirectToSignInAfterFatalApiError(); + return; + } + setNavAvatarLetter(user.email.slice(0, 1).toUpperCase()); + setNavAvatarUrlSafe(null); + } + }, [ + user?.email, + profilesCollection, + appBase.storage, + getCurrentUser, + setNavAvatarUrlSafe, + redirectToSignInAfterFatalApiError, + ]); + + useEffect(() => { + if (!authenticated || !user?.email) return; + void refreshNavAvatar(); + }, [authenticated, user?.email, user?.id, refreshNavAvatar]); + + useEffect(() => { + return () => { + if (navAvatarUrlRef.current) { + URL.revokeObjectURL(navAvatarUrlRef.current); + navAvatarUrlRef.current = null; + } + }; + }, []); + const createTodo = async () => { if (!title.trim()) { setError("Title is required"); @@ -207,7 +293,21 @@ export default function DashboardPage() {

Todo Dashboard

Signed in as {userEmail}

-
+
+ + setProfileModalOpen(false)} + onProfileChanged={() => void refreshNavAvatar()} + /> +

Welcome back

Access your protected dashboard and manage todos.

+ {configHint ? ( +
+

Session was cleared

+

+ The app could not use the API (for example after resetting the database, or if{" "} + NEXT_PUBLIC_APPBASE_API_KEY no + longer matches the server). Create a new API key on the server, update your{" "} + .env, restart the example app, + then sign in again. +

+
+ ) : null}
Home @@ -102,3 +119,17 @@ export default function SignInPage() { ); } + +export default function SignInPage() { + return ( + +

Loading…

+ + } + > + + + ); +} diff --git a/apps/example/app/sign-up/page.tsx b/apps/example/app/sign-up/page.tsx index 936887b..446c952 100644 --- a/apps/example/app/sign-up/page.tsx +++ b/apps/example/app/sign-up/page.tsx @@ -16,6 +16,7 @@ export default function SignUpPage() { const { signUp } = useAuth(); const [email, setEmail] = useState(""); const [password, setPassword] = useState(""); + const [displayName, setDisplayName] = useState(""); const [busy, setBusy] = useState(false); const [logs, setLogs] = useState([]); @@ -29,7 +30,13 @@ export default function SignUpPage() { setBusy(true); push("Sign up started"); try { - const res = await signUp({ email, password }); + const res = await signUp({ + email, + password, + ...(displayName.trim() + ? { customIdentity: { displayName: displayName.trim() } } + : {}), + }); push(`Sign up success: ${res.user.email}`); push("Session active"); router.push("/dashboard"); @@ -69,6 +76,17 @@ export default function SignUpPage() { required /> +