Migration API for importing spaces into a self-hosted install

README.md

@@ -67,6 +67,18 @@ Thank you for considering sponsoring us on GitHub!
 2. [Setup](https://github.com/Lessify/localess/wiki/Setup)
 3. [Integration](https://github.com/Lessify/localess/wiki/Integration)
 
+## Moving to a self-hosted install
+
+A self-hosted Localess install imports spaces from this environment, one space at a time:
+
+1. Deploy this version (functions and hosting) so `/api/migration/**` exists.
+2. In Admin → Settings → Migration, generate a migration token and copy it (it is shown once).
+3. In the self-hosted install, Admin → Spaces → Import from Firebase: enter this environment's URL and the token,
+   pick a space, import. Repeat per space.
+4. Revoke the token when the move is done.
+
+The migration API is read-only and answers `404` while no token is configured.
+
 ## How it works
 
 **Localess** is using Firebase products to run the application.

firebase.json

@@ -63,6 +63,11 @@
         "function": "publicv1",
         "region": "europe-west6"
       },
+      {
+        "source": "/api/migration/**",
+        "function": "migrationapi",
+        "region": "europe-west6"
+      },
       {
         "source": "**",
         "destination": "/index.html"

firestore.rules

@@ -154,8 +154,10 @@ service cloud.firestore {
 
     // Configs
     match /configs/{configId} {
-      allow read: if isSignedIn() && (hasRole('admin') || hasPermission('SETTINGS_MANAGEMENT'));
-      allow write: if isSignedIn() && (hasRole('admin') || hasPermission('SETTINGS_MANAGEMENT'));
+      // configs/migration holds the migration token's hash: only admins read it, and only the admin-only
+      // migrationtoken callables (Admin SDK) write it, so nobody can set a token of their choosing.
+      allow read: if isSignedIn() && (hasRole('admin') || (configId != 'migration' && hasPermission('SETTINGS_MANAGEMENT')));
+      allow write: if isSignedIn() && configId != 'migration' && (hasRole('admin') || hasPermission('SETTINGS_MANAGEMENT'));
     }
   }
 }

functions/src/index.ts

@@ -29,6 +29,10 @@ export { user } from './users';
 
 export { webhook } from './webhooks';
 
+export { migrationapi } from './migration';
+
+export { migrationtoken } from './migration-token';
+
 export { v1 as publicv1 } from './v1';
 
 // Plugins API

functions/src/migration-token.test.ts

@@ -0,0 +1,45 @@
+import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { hashMigrationToken } from './utils/migration-token';
+
+let handlers: { generate: (req: unknown) => Promise<{ token: string }>; revoke: (req: unknown) => Promise<void> };
+let written: unknown;
+let deleted: boolean;
+
+beforeAll(async () => {
+  process.env['FIREBASE_CONFIG'] = JSON.stringify({ projectId: 'test-project', storageBucket: 'test-project.appspot.com' });
+  process.env['GCLOUD_PROJECT'] = 'test-project';
+  const cfg = await import('./config');
+  vi.spyOn(cfg.firestoreService, 'doc').mockImplementation((() => ({
+    set: async (value: unknown) => void (written = value),
+    delete: async () => void (deleted = true),
+  })) as never);
+  const mod = await import('./migration-token');
+  handlers = { generate: mod.migrationtoken.generate.run as never, revoke: mod.migrationtoken.revoke.run as never };
+});
+
+beforeEach(() => {
+  written = undefined;
+  deleted = false;
+});
+
+const admin = { auth: { uid: 'u', token: { role: 'admin' } } };
+const editor = { auth: { uid: 'u', token: { role: 'custom' } } };
+
+describe('migration token callables', () => {
+  it('generates a token for admins and stores only its hash', async () => {
+    const { token } = await handlers.generate({ ...admin, data: {} });
+    expect(token).toMatch(/^[A-Za-z0-9_-]{40}$/);
+    expect(written).toMatchObject({ tokenHash: hashMigrationToken(token) });
+    expect(JSON.stringify(written)).not.toContain(token);
+  });
+
+  it('revokes for admins', async () => {
+    await handlers.revoke({ ...admin, data: {} });
+    expect(deleted).toBe(true);
+  });
+
+  it('refuses everyone else', async () => {
+    await expect(handlers.generate({ ...editor, data: {} })).rejects.toThrow('permission-denied');
+    await expect(handlers.revoke({ data: {} })).rejects.toThrow('permission-denied');
+  });
+});

functions/src/migration-token.ts

@@ -0,0 +1,26 @@
+import { logger } from 'firebase-functions';
+import { HttpsError, onCall } from 'firebase-functions/v2/https';
+import { FieldValue } from 'firebase-admin/firestore';
+import { firestoreService, ROLE_ADMIN } from './config';
+import { hasRole } from './utils/user-auth-utils';
+import { authUid } from './utils/log-auth';
+import { hashMigrationToken, MIGRATION_CONFIG_PATH, newMigrationToken } from './utils/migration-token';
+
+/** Admin → Settings → Migration: a new token replaces the old one; it is returned once and only its hash is kept. */
+const generate = onCall<void, Promise<{ token: string; createdAt: string }>>(async request => {
+  logger.info('[MigrationToken::generate] auth uid: ' + authUid(request.auth));
+  if (!hasRole(ROLE_ADMIN, request.auth)) throw new HttpsError('permission-denied', 'permission-denied');
+  const token = newMigrationToken();
+  const createdAt = new Date();
+  await firestoreService.doc(MIGRATION_CONFIG_PATH).set({ tokenHash: hashMigrationToken(token), createdAt: FieldValue.serverTimestamp() });
+  return { token, createdAt: createdAt.toISOString() };
+});
+
+/** Turns the migration API off (it answers 404 again). */
+const revoke = onCall<void, Promise<void>>(async request => {
+  logger.info('[MigrationToken::revoke] auth uid: ' + authUid(request.auth));
+  if (!hasRole(ROLE_ADMIN, request.auth)) throw new HttpsError('permission-denied', 'permission-denied');
+  await firestoreService.doc(MIGRATION_CONFIG_PATH).delete();
+});
+
+export const migrationtoken = { generate, revoke };

functions/src/migration.test.ts

@@ -0,0 +1,102 @@
+import express from 'express';
+import request from 'supertest';
+import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { hashMigrationToken } from './utils/migration-token';
+
+type Doc = { id: string; data: Record<string, unknown> };
+let collections: Record<string, Doc[]>;
+let config: Record<string, unknown> | undefined;
+let MIGRATION: express.Router;
+let firestoreService: { doc: (path: string) => unknown; collection: (path: string) => unknown };
+
+const ts = (iso: string) => ({ toDate: () => new Date(iso) });
+
+/** A Firestore stand-in for the queries the router makes: doc get, collection orderBy/startAfter/limit/get. */
+function fakeCollection(path: string) {
+  const query = (startAfter?: string, limit?: number) => ({
+    orderBy: () => query(startAfter, limit),
+    startAfter: (id: string) => query(id, limit),
+    limit: (n: number) => query(startAfter, n),
+    get: async () => {
+      const all = [...(collections[path] ?? [])].sort((a, b) => (a.id < b.id ? -1 : 1));
+      const from = startAfter ? all.filter(it => it.id > startAfter) : all;
+      const docs = (limit ? from.slice(0, limit) : from).map(it => ({ id: it.id, data: () => it.data }));
+      return { docs, empty: docs.length === 0 };
+    },
+  });
+  return query();
+}
+
+beforeAll(async () => {
+  process.env['FIREBASE_CONFIG'] = JSON.stringify({ projectId: 'test-project', storageBucket: 'test-project.appspot.com' });
+  process.env['GCLOUD_PROJECT'] = 'test-project';
+  const cfg = await import('./config');
+  firestoreService = cfg.firestoreService as never;
+  MIGRATION = (await import('./migration')).MIGRATION;
+});
+
+beforeEach(() => {
+  vi.restoreAllMocks();
+  config = { tokenHash: hashMigrationToken('secret') };
+  collections = {
+    spaces: [{ id: 's1', data: { name: 'Site', locales: [{ id: 'en', name: 'English' }], localeFallback: { id: 'en', name: 'English' }, createdAt: ts('2025-01-01T00:00:00Z') } }],
+    'spaces/s1/tokens': [{ id: 'TOKEN000000000000001', data: { name: 'web', createdAt: ts('2025-01-02T00:00:00Z') } }],
+    'spaces/s1/webhooks': [],
+    'spaces/s1/schemas': [{ id: 'page', data: { type: 'ROOT', fields: [] } }],
+    'spaces/s1/translations': Array.from({ length: 501 }, (_, i) => ({ id: `k${String(i).padStart(4, '0')}`, data: { type: 'STRING', locales: { en: 'x' } } })),
+    'spaces/s1/assets': [],
+    'spaces/s1/contents': [{ id: 'c1', data: { kind: 'DOCUMENT', data: '{"_id":"r"}' } }],
+  };
+  vi.spyOn(firestoreService, 'doc').mockImplementation(((path: string) => ({
+    get: async () => {
+      if (path === 'configs/migration') return { exists: !!config, data: () => config };
+      const [, id] = path.split('/');
+      const found = collections['spaces'].find(it => it.id === id);
+      return { exists: !!found, id, data: () => found?.data };
+    },
+  })) as never);
+  vi.spyOn(firestoreService, 'collection').mockImplementation(((path: string) => fakeCollection(path)) as never);
+});
+
+const app = () => express().use(MIGRATION);
+const get = (url: string, token: string | null = 'secret') => {
+  const req = request(app()).get(url);
+  return token ? req.set('Authorization', `Bearer ${token}`) : req;
+};
+
+describe('migration API', () => {
+  it('answers 404 everywhere while no token is configured', async () => {
+    config = undefined;
+    expect((await get('/api/migration/spaces')).status).toBe(404);
+  });
+
+  it('answers 401 without or with a wrong token', async () => {
+    expect((await get('/api/migration/spaces', null)).status).toBe(401);
+    expect((await get('/api/migration/spaces', 'wrong')).status).toBe(401);
+  });
+
+  it('lists spaces and returns one with ISO timestamps', async () => {
+    expect((await get('/api/migration/spaces')).body).toEqual([{ id: 's1', name: 'Site', createdAt: '2025-01-01T00:00:00.000Z' }]);
+    expect((await get('/api/migration/spaces/s1')).body).toMatchObject({ id: 's1', name: 'Site', createdAt: '2025-01-01T00:00:00.000Z' });
+    expect((await get('/api/migration/spaces/nope')).status).toBe(404);
+  });
+
+  it('returns small collections whole, with ids', async () => {
+    expect((await get('/api/migration/spaces/s1/tokens')).body).toEqual([{ id: 'TOKEN000000000000001', name: 'web', createdAt: '2025-01-02T00:00:00.000Z' }]);
+    expect((await get('/api/migration/spaces/s1/schemas')).body).toEqual([{ id: 'page', type: 'ROOT', fields: [] }]);
+    expect((await get('/api/migration/spaces/s1/webhooks')).body).toEqual([]);
+  });
+
+  it('pages large collections by 500, cursor = last id, null at the end', async () => {
+    const first = (await get('/api/migration/spaces/s1/translations')).body;
+    expect(first.items).toHaveLength(500);
+    expect(first.cursor).toBe('k0499');
+    const second = (await get('/api/migration/spaces/s1/translations?cursor=k0499')).body;
+    expect(second.items.map((it: { id: string }) => it.id)).toEqual(['k0500']);
+    expect(second.cursor).toBeNull();
+  });
+
+  it('returns contents as stored, data string included', async () => {
+    expect((await get('/api/migration/spaces/s1/contents')).body).toEqual({ items: [{ id: 'c1', kind: 'DOCUMENT', data: '{"_id":"r"}' }], cursor: null });
+  });
+});

functions/src/migration.ts

@@ -0,0 +1,107 @@
+import express, { NextFunction, Request, Response } from 'express';
+import { onRequest } from 'firebase-functions/v2/https';
+import { firestoreService } from './config';
+import { checkMigrationToken, MIGRATION_CONFIG_PATH, MigrationConfig } from './utils/migration-token';
+
+/**
+ * Read-only API a self-hosted Localess install calls to import a space (Admin → Spaces → Import from Firebase).
+ * Disabled (404) until an admin generates a migration token in Admin → Settings → Migration.
+ */
+const PAGE_SIZE = 500;
+
+/**
+ * Firestore Timestamps (at any depth) → ISO strings; everything else as stored.
+ * @param {unknown} value a Firestore value
+ * @return {unknown} its JSON form
+ */
+function plain(value: unknown): unknown {
+  if (value && typeof value === 'object') {
+    const ts = value as { toDate?: () => Date };
+    if (typeof ts.toDate === 'function') return ts.toDate().toISOString();
+    if (Array.isArray(value)) return value.map(plain);
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, plain(item)]));
+  }
+  return value;
+}
+
+const toJson = (doc: { id: string; data: () => unknown }) => ({ id: doc.id, ...(plain(doc.data()) as object) });
+
+/**
+ * Answers 404 while no migration token is configured, 401 for a missing or wrong one.
+ * @param {Request} req request
+ * @param {Response} res response
+ * @param {NextFunction} next next handler
+ */
+async function auth(req: Request, res: Response, next: NextFunction): Promise<void> {
+  const config = (await firestoreService.doc(MIGRATION_CONFIG_PATH).get()).data() as MigrationConfig | undefined;
+  const result = checkMigrationToken(req.header('authorization'), config);
+  if (result === 'disabled') {
+    res.status(404).json({ message: 'Not found' });
+    return;
+  }
+  if (result === 'unauthorized') {
+    res.status(401).json({ message: 'Missing or invalid migration token' });
+    return;
+  }
+  next();
+}
+
+/**
+ * Answers 404 for an unknown space.
+ * @param {Request} req request
+ * @param {Response} res response
+ * @return {Promise<boolean>} whether the space exists
+ */
+async function requireSpace(req: Request, res: Response): Promise<boolean> {
+  const doc = await firestoreService.doc(`spaces/${req.params['spaceId']}`).get();
+  if (!doc.exists) res.status(404).json({ message: 'Space not found' });
+  return doc.exists;
+}
+
+// eslint-disable-next-line new-cap
+export const MIGRATION = express.Router();
+MIGRATION.use('/api/migration', auth);
+
+MIGRATION.get('/api/migration/spaces', async (_req, res) => {
+  const snapshot = await firestoreService.collection('spaces').orderBy('__name__').get();
+  res.json(
+    snapshot.docs.map(doc => {
+      const space = toJson(doc) as { id: string; name?: string; createdAt?: string };
+      return { id: space.id, name: space.name, createdAt: space.createdAt };
+    })
+  );
+});
+
+MIGRATION.get('/api/migration/spaces/:spaceId', async (req, res) => {
+  const doc = await firestoreService.doc(`spaces/${req.params['spaceId']}`).get();
+  if (!doc.exists) {
+    res.status(404).json({ message: 'Space not found' });
+    return;
+  }
+  res.json(toJson({ id: doc.id, data: () => doc.data() }));
+});
+
+for (const name of ['tokens', 'webhooks', 'schemas']) {
+  MIGRATION.get(`/api/migration/spaces/:spaceId/${name}`, async (req, res) => {
+    if (!(await requireSpace(req, res))) return;
+    const snapshot = await firestoreService.collection(`spaces/${req.params['spaceId']}/${name}`).orderBy('__name__').get();
+    res.json(snapshot.docs.map(toJson));
+  });
+}
+
+for (const name of ['translations', 'assets', 'contents']) {
+  MIGRATION.get(`/api/migration/spaces/:spaceId/${name}`, async (req, res) => {
+    if (!(await requireSpace(req, res))) return;
+    let query = firestoreService.collection(`spaces/${req.params['spaceId']}/${name}`).orderBy('__name__');
+    const cursor = typeof req.query['cursor'] === 'string' ? req.query['cursor'] : undefined;
+    if (cursor) query = query.startAfter(cursor);
+    const snapshot = await query.limit(PAGE_SIZE).get();
+    const items = snapshot.docs.map(toJson);
+    res.json({ items, cursor: items.length === PAGE_SIZE ? items[items.length - 1].id : null });
+  });
+}
+
+const app = express();
+app.use(MIGRATION);
+
+export const migrationapi = onRequest({ memory: '512MiB', maxInstances: 2 }, app);

functions/src/utils/migration-token.test.ts

@@ -0,0 +1,27 @@
+import { describe, expect, it } from 'vitest';
+import { checkMigrationToken, hashMigrationToken, newMigrationToken } from './migration-token';
+
+describe('migration token', () => {
+  it('makes 40 url-safe characters, different every time', () => {
+    const a = newMigrationToken();
+    expect(a).toMatch(/^[A-Za-z0-9_-]{40}$/);
+    expect(newMigrationToken()).not.toBe(a);
+  });
+
+  it('hashes with sha256 hex', () => {
+    expect(hashMigrationToken('abc')).toBe('ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad');
+  });
+
+  it('is disabled without a stored hash', () => {
+    expect(checkMigrationToken('Bearer x', undefined)).toBe('disabled');
+    expect(checkMigrationToken('Bearer x', {})).toBe('disabled');
+  });
+
+  it('accepts only the matching bearer token', () => {
+    const stored = { tokenHash: hashMigrationToken('secret') };
+    expect(checkMigrationToken('Bearer secret', stored)).toBe('ok');
+    expect(checkMigrationToken('Bearer other', stored)).toBe('unauthorized');
+    expect(checkMigrationToken('secret', stored)).toBe('unauthorized');
+    expect(checkMigrationToken(undefined, stored)).toBe('unauthorized');
+  });
+});