From 760fe63cd69df3f7ad0c59296db2e3d0af53bbd6 Mon Sep 17 00:00:00 2001 From: Tunir Date: Thu, 28 May 2026 08:58:34 +0530 Subject: [PATCH] docs: add db restore disaster recovery runbook --- backend/README.md | 8 + backend/docs/DISASTER_RECOVERY_DB_RESTORE.md | 176 ++++++++++++++ backend/package.json | 1 + backend/scripts/README.md | 25 ++ backend/scripts/verify-db-restore.js | 231 +++++++++++++++++++ 5 files changed, 441 insertions(+) create mode 100644 backend/docs/DISASTER_RECOVERY_DB_RESTORE.md create mode 100644 backend/scripts/verify-db-restore.js diff --git a/backend/README.md b/backend/README.md index 3c6280e..aa21599 100644 --- a/backend/README.md +++ b/backend/README.md @@ -57,6 +57,14 @@ To run the linter: npm run lint ``` +### Database Restore Verification +To verify that the SQLite backend database can be backed up and restored without mutating the source database: +```bash +npm run db:restore:verify +``` + +For the full disaster recovery procedure, see [Database Restore Disaster Recovery Runbook](./docs/DISASTER_RECOVERY_DB_RESTORE.md). + ## External KYC Providers (SEP-12) AnchorPoint supports pluggable third-party KYC providers for SEP-12 flows. diff --git a/backend/docs/DISASTER_RECOVERY_DB_RESTORE.md b/backend/docs/DISASTER_RECOVERY_DB_RESTORE.md new file mode 100644 index 0000000..26fee69 --- /dev/null +++ b/backend/docs/DISASTER_RECOVERY_DB_RESTORE.md @@ -0,0 +1,176 @@ +# Database Restore Disaster Recovery Runbook + +This runbook verifies that AnchorPoint can restore its SQLite-backed backend database before testnet deployment. It is intentionally focused on database availability, integrity, and operational safety; it does not print secrets or row-level customer data. + +## Scope + +- Backend database configured by `DATABASE_URL`. +- Docker Compose deployment using the `backend-data` volume. +- Local development database at `backend/prisma/dev.db`. +- Restore validation for Prisma-managed tables, KYC records, API keys, transactions, recurring payments, notifications, and system configuration. + +Redis, Jaeger, and Prometheus data are out of scope for this procedure. They should be recreated or restored through their own service-specific backup plans. + +## Recovery Objectives + +| Objective | Target | +| --- | --- | +| RPO | Latest verified database backup | +| RTO | Under 30 minutes for local/testnet SQLite restore | +| Integrity gate | `PRAGMA quick_check` returns `ok` | +| Data gate | Restored table list and row counts match the backup source | +| App gate | Backend health endpoint responds after restart | + +## Prerequisites + +- Access to the deployment host or local checkout. +- `sqlite3` CLI installed on the host running the verification. +- Node.js and npm installed for the backend verification helper. +- A recent backup file stored outside the application data volume. +- No private keys, JWT secrets, KYC documents, or API key values pasted into logs or issue comments. + +## Local Verification Command + +Run from `backend/`: + +```bash +npm run db:restore:verify +``` + +The command reads `DATABASE_URL` when set. If `DATABASE_URL` is absent, it verifies `file:./prisma/dev.db`. + +To verify a specific SQLite file: + +```bash +npm run db:restore:verify -- --source ./prisma/dev.db +``` + +To place evidence files in a known directory: + +```bash +npm run db:restore:verify -- --source ./prisma/dev.db --backup-dir ./tmp/dr-restore +``` + +The helper performs these checks: + +- Confirms the source database exists. +- Runs `PRAGMA quick_check` on the source database. +- Creates a SQLite backup using the SQLite online backup command. +- Restores the backup into an isolated probe database. +- Runs `PRAGMA quick_check` on the backup and restored probe. +- Compares user table names and row counts between the source and restored probe. + +## Docker Testnet Backup Procedure + +Create a host-side backup directory: + +```bash +mkdir -p backups +``` + +Create a backup from the `backend-data` Docker volume: + +```bash +docker run --rm \ + -v anchorpoint_backend-data:/data \ + -v "$PWD/backups:/backups" \ + alpine:3.20 \ + sh -lc 'apk add --no-cache sqlite >/dev/null && sqlite3 /data/dev.db ".backup /backups/anchorpoint-$(date -u +%Y%m%dT%H%M%SZ).db"' +``` + +Verify the newest backup: + +```bash +sqlite3 backups/.db "PRAGMA quick_check;" +``` + +Expected output: + +```text +ok +``` + +## Docker Testnet Restore Procedure + +Pause writes before restoring. For the Compose deployment, stop the backend while leaving Redis and observability services available: + +```bash +docker compose stop backend +``` + +Preserve the current database before replacing it: + +```bash +docker run --rm \ + -v anchorpoint_backend-data:/data \ + -v "$PWD/backups:/backups" \ + alpine:3.20 \ + sh -lc 'apk add --no-cache sqlite >/dev/null && cp /data/dev.db /backups/pre-restore-$(date -u +%Y%m%dT%H%M%SZ).db' +``` + +Restore the selected backup: + +```bash +docker run --rm \ + -v anchorpoint_backend-data:/data \ + -v "$PWD/backups:/backups" \ + alpine:3.20 \ + sh -lc 'cp /backups/.db /data/dev.db && chmod 600 /data/dev.db' +``` + +Start the backend: + +```bash +docker compose up -d backend +``` + +Validate service health: + +```bash +curl -fsS http://localhost:3002/health +``` + +Run migration and restore checks from the backend workspace when dependencies are available: + +```bash +npm run migrate:status +npm run db:restore:verify -- --source ./prisma/dev.db --backup-dir ./tmp/dr-restore +``` + +## Post-Restore QA Checklist + +- `sqlite3 "PRAGMA quick_check;"` returns `ok`. +- Table names match the source backup. +- Row counts match the source backup for all application tables. +- Backend `/health` returns success after restart. +- Logs do not include private keys, JWT secrets, API key values, KYC documents, or full customer payloads. +- Recent transaction, KYC, API key, recurring payment, and notification records are visible through normal application paths. +- Any failed restore attempt has been rolled back using the preserved `pre-restore-*` copy. + +## Failure and Rollback + +If the backend does not start or validation fails: + +1. Stop the backend. +2. Replace `/data/dev.db` with the `pre-restore-*` copy. +3. Restart the backend. +4. Confirm `/health` responds. +5. Capture the failing command, exit code, and sanitized logs. + +Do not continue with testnet deployment until the restore probe passes or the failed restore is explicitly accepted by maintainers. + +## PR Evidence Template + +Use this format in pull requests or release notes: + +```text +DB restore verification +- Source: +- Backup path: +- Restore probe path: +- PRAGMA quick_check: ok +- Tables verified: +- Rows verified: +- Health check: +- Notes: +``` diff --git a/backend/package.json b/backend/package.json index 97666a0..9f79ab5 100644 --- a/backend/package.json +++ b/backend/package.json @@ -21,6 +21,7 @@ "prisma:migrate": "prisma migrate dev", "prisma:deploy": "prisma migrate deploy", "prisma:studio": "prisma studio", + "db:restore:verify": "node scripts/verify-db-restore.js", "migrate:verify": "ts-node scripts/verify-migrations.ts", "migrate:check": "ts-node scripts/migration-integrity-checker.ts", "migrate:rollback": "ts-node scripts/generate-rollback.ts", diff --git a/backend/scripts/README.md b/backend/scripts/README.md index c947be8..723a78b 100644 --- a/backend/scripts/README.md +++ b/backend/scripts/README.md @@ -45,6 +45,28 @@ Creates a `rollback.sql` file in the migration directory with: - Confidence levels (high/medium/low) - Manual intervention notes where needed +### 3. verify-db-restore.js + +Verifies that the SQLite database can be backed up and restored without mutating the source database. + +**Usage:** +```bash +npm run db:restore:verify + +# Verify a specific database file +npm run db:restore:verify -- --source ./prisma/dev.db + +# Keep backup and restore probe files in a known directory +npm run db:restore:verify -- --source ./prisma/dev.db --backup-dir ./tmp/dr-restore +``` + +**Features:** +- Runs `PRAGMA quick_check` on the source, backup, and restored probe. +- Creates a SQLite backup with the online backup command. +- Restores into an isolated probe database. +- Compares user table names and row counts. +- Avoids printing row-level data or secrets. + **Confidence Levels:** - **High**: Automatic rollback possible (e.g., DROP TABLE for CREATE TABLE) - **Medium**: Rollback possible with review (e.g., recreate index) @@ -57,6 +79,9 @@ Creates a `rollback.sql` file in the migration directory with: ```bash cd backend +# Verify database backup/restore readiness +npm run db:restore:verify + # Check migration integrity npm run migrate:check diff --git a/backend/scripts/verify-db-restore.js b/backend/scripts/verify-db-restore.js new file mode 100644 index 0000000..2b28808 --- /dev/null +++ b/backend/scripts/verify-db-restore.js @@ -0,0 +1,231 @@ +#!/usr/bin/env node + +const { execFileSync } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const DEFAULT_DATABASE_URL = 'file:./prisma/dev.db'; + +function parseArgs(argv) { + const args = { + source: undefined, + databaseUrl: process.env.DATABASE_URL || DEFAULT_DATABASE_URL, + backupDir: path.join(os.tmpdir(), 'anchorpoint-db-restore'), + restoreTarget: undefined, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = argv[index + 1]; + + if (arg === '--source') { + args.source = requireValue(arg, next); + index += 1; + } else if (arg === '--database-url') { + args.databaseUrl = requireValue(arg, next); + index += 1; + } else if (arg === '--backup-dir') { + args.backupDir = requireValue(arg, next); + index += 1; + } else if (arg === '--restore-target') { + args.restoreTarget = requireValue(arg, next); + index += 1; + } else if (arg === '--help' || arg === '-h') { + printHelp(); + process.exit(0); + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + + return args; +} + +function requireValue(arg, value) { + if (!value || value.startsWith('--')) { + throw new Error(`${arg} requires a value`); + } + + return value; +} + +function printHelp() { + console.log(` +Verify that an AnchorPoint SQLite database can be backed up and restored. + +Usage: + npm run db:restore:verify + npm run db:restore:verify -- --source ./prisma/dev.db + npm run db:restore:verify -- --database-url file:./prisma/dev.db --backup-dir /tmp/anchorpoint-dr + +Options: + --source SQLite database file to verify. Overrides DATABASE_URL. + --database-url Prisma SQLite DATABASE_URL. Only file: URLs are supported. + --backup-dir Directory for generated backup and restore probe files. + --restore-target Optional explicit restore target path. +`); +} + +function resolveDatabasePath(args) { + if (args.source) { + return path.resolve(process.cwd(), args.source); + } + + const databaseUrl = args.databaseUrl; + if (!databaseUrl.startsWith('file:')) { + throw new Error('Only SQLite file: DATABASE_URL values are supported by this restore verifier'); + } + + const rawPath = databaseUrl.slice('file:'.length); + if (!rawPath) { + throw new Error('DATABASE_URL file path is empty'); + } + + return path.isAbsolute(rawPath) ? rawPath : path.resolve(process.cwd(), rawPath); +} + +function ensureSqliteAvailable() { + try { + execFileSync('sqlite3', ['--version'], { stdio: 'pipe' }); + } catch (error) { + throw new Error('sqlite3 CLI is required to run restore verification'); + } +} + +function runSqlite(dbPath, sql) { + return execFileSync('sqlite3', [dbPath, sql], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); +} + +function quoteIdentifier(identifier) { + return `"${identifier.replace(/"/g, '""')}"`; +} + +function quoteDotCommandPath(filePath) { + return `'${filePath.replace(/'/g, "''")}'`; +} + +function quickCheck(dbPath) { + const result = runSqlite(dbPath, 'PRAGMA quick_check;'); + if (result !== 'ok') { + throw new Error(`SQLite quick_check failed for ${dbPath}: ${result}`); + } +} + +function listTables(dbPath) { + const output = runSqlite( + dbPath, + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name;" + ); + + return output ? output.split(/\r?\n/).filter(Boolean) : []; +} + +function tableCounts(dbPath, tables) { + return tables.map((table) => { + const count = runSqlite(dbPath, `SELECT COUNT(*) FROM ${quoteIdentifier(table)};`); + return { table, count: Number(count) }; + }); +} + +function assertMatchingCounts(sourceCounts, restoredCounts) { + const sourceByTable = new Map(sourceCounts.map((entry) => [entry.table, entry.count])); + const restoredByTable = new Map(restoredCounts.map((entry) => [entry.table, entry.count])); + + for (const table of sourceByTable.keys()) { + if (!restoredByTable.has(table)) { + throw new Error(`Restored database is missing table ${table}`); + } + + if (sourceByTable.get(table) !== restoredByTable.get(table)) { + throw new Error( + `Row count mismatch for ${table}: source=${sourceByTable.get(table)} restored=${restoredByTable.get(table)}` + ); + } + } + + for (const table of restoredByTable.keys()) { + if (!sourceByTable.has(table)) { + throw new Error(`Restored database has unexpected table ${table}`); + } + } +} + +function createBackup(sourcePath, backupPath) { + fs.mkdirSync(path.dirname(backupPath), { recursive: true }); + if (fs.existsSync(backupPath)) { + fs.unlinkSync(backupPath); + } + + runSqlite(sourcePath, `.backup ${quoteDotCommandPath(backupPath)}`); +} + +function copyBackupToRestoreTarget(backupPath, restoreTarget) { + fs.mkdirSync(path.dirname(restoreTarget), { recursive: true }); + if (fs.existsSync(restoreTarget)) { + fs.unlinkSync(restoreTarget); + } + + fs.copyFileSync(backupPath, restoreTarget); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const sourcePath = resolveDatabasePath(args); + + if (!fs.existsSync(sourcePath)) { + throw new Error(`Database file does not exist: ${sourcePath}`); + } + + ensureSqliteAvailable(); + + const timestamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..+/, 'Z'); + const backupPath = path.resolve(args.backupDir, `anchorpoint-${timestamp}.backup.db`); + const restoreTarget = args.restoreTarget + ? path.resolve(process.cwd(), args.restoreTarget) + : path.resolve(args.backupDir, `anchorpoint-${timestamp}.restore-check.db`); + + quickCheck(sourcePath); + const sourceTables = listTables(sourcePath); + const sourceCounts = tableCounts(sourcePath, sourceTables); + + createBackup(sourcePath, backupPath); + quickCheck(backupPath); + + copyBackupToRestoreTarget(backupPath, restoreTarget); + quickCheck(restoreTarget); + + const restoredTables = listTables(restoreTarget); + const restoredCounts = tableCounts(restoreTarget, restoredTables); + assertMatchingCounts(sourceCounts, restoredCounts); + + const totalRows = sourceCounts.reduce((sum, entry) => sum + entry.count, 0); + + console.log('AnchorPoint DB restore verification passed'); + console.log(`Source: ${sourcePath}`); + console.log(`Backup: ${backupPath}`); + console.log(`Restore probe: ${restoreTarget}`); + console.log(`Tables verified: ${sourceTables.length}`); + console.log(`Rows verified: ${totalRows}`); +} + +if (require.main === module) { + try { + main(); + } catch (error) { + console.error(`AnchorPoint DB restore verification failed: ${error.message}`); + process.exit(1); + } +} + +module.exports = { + parseArgs, + resolveDatabasePath, + quoteIdentifier, + listTables, + tableCounts, + assertMatchingCounts, +};