From 19f1df2092810044248ea617ee3958ff39907682 Mon Sep 17 00:00:00 2001 From: aboyejirebecca-prog Date: Sat, 30 May 2026 21:51:42 +0100 Subject: [PATCH] Stringhtened migration drift checks across environmtents --- .github/workflows/database.yml | 20 + .github/workflows/migration-drift-check.yml | 256 +++++++- backend/package.json | 3 + backend/tests/migration-drift.test.ts | 611 ++++++++++++++++++++ docs/MIGRATION_DRIFT_CHECKING.md | 318 ++++++++++ docs/MIGRATION_DRIFT_QUICK_REFERENCE.md | 113 ++++ docs/MIGRATION_REMEDIATION.md | 434 ++++++++++++++ scripts/check-migration-drift.js | 330 +++++++++-- scripts/validate-migration-state.js | 287 +++++++++ scripts/validate-script-syntax.js | 23 + 10 files changed, 2333 insertions(+), 62 deletions(-) create mode 100644 backend/tests/migration-drift.test.ts create mode 100644 docs/MIGRATION_DRIFT_CHECKING.md create mode 100644 docs/MIGRATION_DRIFT_QUICK_REFERENCE.md create mode 100644 docs/MIGRATION_REMEDIATION.md create mode 100644 scripts/validate-migration-state.js create mode 100644 scripts/validate-script-syntax.js diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index c06d4e80..f1ab145b 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -46,6 +46,26 @@ jobs: working-directory: backend run: npm ci + - name: Run migration drift check against local database + id: local-drift-check + # Official Supabase demo service role key ā€” only valid for localhost:54321 + env: + SUPABASE_URL: http://localhost:54321 + SUPABASE_SERVICE_ROLE_KEY: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU + run: | + echo "šŸ” Verifying migration state against local database..." + node scripts/check-migration-drift.js --verify-db --env local --json > local-drift-result.json 2>&1 || true + cat local-drift-result.json + node -e " + const r = JSON.parse(require('fs').readFileSync('local-drift-result.json', 'utf8')); + if (!r.success) { + console.error('āŒ Local database drift detected:'); + (r.dbCheck?.issues || []).forEach(i => console.error(' [' + i.type + '] ' + i.message)); + process.exit(1); + } + console.log('āœ… Local database migration state matches filesystem (' + (r.dbCheck?.appliedCount ?? '?') + ' applied)'); + " + - name: Run RLS Policy Audit env: SUPABASE_URL: http://localhost:54321 diff --git a/.github/workflows/migration-drift-check.yml b/.github/workflows/migration-drift-check.yml index e8aafce8..e804c450 100644 --- a/.github/workflows/migration-drift-check.yml +++ b/.github/workflows/migration-drift-check.yml @@ -32,38 +32,254 @@ jobs: - name: Run migration drift check id: drift-check run: | - if node scripts/check-migration-drift.js; then - echo "status=success" >> $GITHUB_OUTPUT - echo "āœ… No migration drift detected" - else - echo "status=failed" >> $GITHUB_OUTPUT - echo "āŒ Migration drift detected!" - exit 1 - fi + node scripts/check-migration-drift.js --json > drift-result.json 2>&1 || true + cat drift-result.json + node -e "process.exit(JSON.parse(require('fs').readFileSync('drift-result.json','utf8')).success ? 0 : 1)" - name: Comment on PR - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' && always() uses: actions/github-script@v9 with: script: | + const fs = require('fs'); + let result; + try { + result = JSON.parse(fs.readFileSync('drift-result.json', 'utf-8')); + } catch { + console.log('āš ļø Could not parse drift check result'); + return; + } + + const status = result.success ? 'āœ… No drift detected' : 'āŒ Drift detected!'; + let details = ''; + if (result.issues?.length > 0) { + const errors = result.issues.filter(i => i.severity === 'error'); + const warnings = result.issues.filter(i => i.severity === 'warning'); + if (errors.length > 0) { + details += '\n\n**Errors (must fix before merging):**\n' + + errors.map(i => `- \`[${i.type}]\` ${i.message}`).join('\n'); + } + if (warnings.length > 0) { + details += '\n\n**Warnings (review recommended):**\n' + + warnings.map(i => `- \`[${i.type}]\` ${i.message}`).join('\n'); + } + } + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number + }); + + const existingComment = comments.find(c => c.body.includes('## šŸ” Migration Drift Check')); + + const body = `## šŸ” Migration Drift Check\n**Result**: ${status}${details}\n\nFor guidance on fixing drift issues, see [docs/MIGRATION_REMEDIATION.md](../../docs/MIGRATION_REMEDIATION.md)`; + + if (existingComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body + }); + } + + verify-local-database: + name: Verify local database migration state + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup Supabase CLI + uses: supabase/setup-cli@v2 + with: + version: latest + + - name: Start Supabase local stack + run: supabase start + + - name: Apply all migrations + run: supabase db push + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run drift check against local database + id: local-drift-check + # Official Supabase demo service role key ā€” only valid for localhost:54321 + env: + SUPABASE_URL: http://localhost:54321 + SUPABASE_SERVICE_ROLE_KEY: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU + run: | + node scripts/check-migration-drift.js --verify-db --env local --json > local-drift-result.json 2>&1 || true + cat local-drift-result.json + node -e "process.exit(JSON.parse(require('fs').readFileSync('local-drift-result.json','utf8')).success ? 0 : 1)" + + - name: Comment local verification on PR + if: github.event_name == 'pull_request' && always() + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + let result; + try { + result = JSON.parse(fs.readFileSync('local-drift-result.json', 'utf-8')); + } catch { return; } + + const status = result.success ? 'āœ… Verified' : 'āš ļø Issues detected'; + let details = ''; + if (result.dbCheck?.issues?.length > 0) { + const errors = result.dbCheck.issues.filter(i => i.severity === 'error'); + const warnings = result.dbCheck.issues.filter(i => i.severity === 'warning'); + if (errors.length > 0) { + details += '\n\n**Errors:**\n' + + errors.map(i => `- \`[${i.type}]\` ${i.message}`).join('\n'); + } + if (warnings.length > 0) { + details += '\n\n**Warnings:**\n' + + warnings.map(i => `- \`[${i.type}]\` ${i.message}${i.executedAt ? ` _(applied ${i.executedAt})_` : ''}`).join('\n'); + } + } + const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number }); - - const driftComment = comments.find(c => c.body.includes('Migration Drift Check')); - - const body = `## šŸ” Migration Drift Check - **Result**: ${process.env.STATUS === 'success' ? 'āœ… No drift detected' : 'āŒ Drift detected!'} - - Please review the migration files in both \`backend/migrations\` and \`supabase/migrations\` folders.`; - - if (driftComment) { + const existingComment = comments.find(c => c.body.includes('## šŸ–„ļø Local Database Drift Check')); + + const counts = result.dbCheck + ? `\nApplied: ${result.dbCheck.appliedCount} | Filesystem: ${result.dbCheck.filesystemCount}` + : ''; + const body = `## šŸ–„ļø Local Database Drift Check\n**Result**: ${status}${counts}${details}\n\nFor guidance, see [docs/MIGRATION_REMEDIATION.md](../../docs/MIGRATION_REMEDIATION.md)`; + + if (existingComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body + }); + } + + - name: Stop Supabase local stack + if: always() + run: supabase stop + + verify-database: + name: Verify CI remote database migration state + runs-on: ubuntu-latest + # Only run when remote database credentials are available + if: github.event_name == 'push' || (github.event_name == 'pull_request' && vars.ENABLE_DB_VERIFICATION == 'true') + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Verify applied migrations against CI remote database + id: db-verify + run: | + node scripts/check-migration-drift.js --verify-db --env ci-remote --json > drift-result.json 2>&1 + cat drift-result.json + env: + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} + continue-on-error: true + + - name: Parse and report results + if: always() + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + let result; + try { + result = JSON.parse(fs.readFileSync('drift-result.json', 'utf-8')); + } catch { + console.log('āš ļø Could not parse drift check result'); + return; + } + + if (!result.success) { + let dbIssues = ''; + if (result.dbCheck?.issues?.length > 0) { + dbIssues = result.dbCheck.issues + .map(i => `- [${i.type}] ${i.message}`) + .join('\n'); + } + core.setFailed(`CI remote database verification failed:\n${dbIssues || 'See logs for details'}`); + } else { + console.log('āœ… CI remote database migration state verified'); + if (result.dbCheck?.appliedCount != null) { + console.log(`Applied migrations: ${result.dbCheck.appliedCount}`); + } + } + + - name: Comment database verification result + if: github.event_name == 'pull_request' && always() + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + let result; + try { + result = JSON.parse(fs.readFileSync('drift-result.json', 'utf-8')); + } catch { return; } + + const status = result.success ? 'āœ… Verified' : 'āš ļø Issues detected'; + let details = ''; + if (result.dbCheck?.issues?.length > 0) { + details = '\n\n**Issues:**\n' + + result.dbCheck.issues.map(i => `- \`[${i.type}]\` ${i.message}`).join('\n'); + } + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number + }); + + const existingComment = comments.find(c => c.body.includes('## šŸ“Š CI Remote Database Drift Check')); + + const counts = result.dbCheck + ? `\nApplied: ${result.dbCheck.appliedCount} | Filesystem: ${result.dbCheck.filesystemCount}` + : ''; + const body = `## šŸ“Š CI Remote Database Drift Check\n**Result**: ${status}${counts}${details}\n\nFor more information, see [docs/MIGRATION_REMEDIATION.md](../../docs/MIGRATION_REMEDIATION.md)`; + + if (existingComment) { await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, - comment_id: driftComment.id, + comment_id: existingComment.id, body }); } else { @@ -73,4 +289,4 @@ jobs: issue_number: context.issue.number, body }); - } \ No newline at end of file + } diff --git a/backend/package.json b/backend/package.json index ff58d16d..0028e8f5 100644 --- a/backend/package.json +++ b/backend/package.json @@ -14,6 +14,7 @@ "dev": "ts-node-dev --respawn --transpile-only src/index.ts", "validate-env": "node scripts/validate-env.js", "test": "jest", + "test:migrations": "jest migration-drift.test.ts", "test:smoke": "jest -c tests/smoke/jest.smoke.config.js", "test:smoke:verbose": "VERBOSE=1 jest -c tests/smoke/jest.smoke.config.js --verbose", "setup:smoke-user": "ts-node ../scripts/setup-smoke-test-user.ts", @@ -26,6 +27,8 @@ "audit:rls:local": "node ../scripts/run-rls-audit-local.js", "test:rls-audit": "node ../scripts/test-rls-audit.js", "check:migrations": "node ../scripts/check-migration-drift.js", + "check:migrations:verify-db": "node ../scripts/check-migration-drift.js --verify-db", + "validate:migration-state": "node ../scripts/validate-migration-state.js", "db:verify-rollback": "node ../scripts/verify-rollback-safety.js", "db:verify-rollback:group": "node ../scripts/verify-rollback-safety.js --group", "validate:env": "ts-node scripts/validate-env.ts" diff --git a/backend/tests/migration-drift.test.ts b/backend/tests/migration-drift.test.ts new file mode 100644 index 00000000..d1ae6ddb --- /dev/null +++ b/backend/tests/migration-drift.test.ts @@ -0,0 +1,611 @@ +import { afterEach, beforeEach, describe, expect, it } from '@jest/globals'; +import fs from 'fs'; +import path from 'path'; + +// Mock module - we'll test the core logic functions +const BACKEND_MIGRATIONS = path.join(__dirname, '..', '..', 'backend', 'migrations'); +const SUPABASE_MIGRATIONS = path.join(__dirname, '..', '..', 'supabase', 'migrations'); + +// Core utility functions (extracted from drift check script for testing) +function normalizeSQL(content: string): string { + return content + .replace(/--.*$/gm, '') // Remove single-line comments + .replace(/\/\*[\s\S]*?\*\//g, '') // Remove multi-line comments + .replace(/\s+/g, ' ') // Normalize whitespace + .toLowerCase() + .trim(); +} + +function extractTables(sql: string): Set { + const tableRegex = /create\s+table\s+(?:if\s+not\s+exists\s+)?(?:public\.)?(\w+)/gi; + const alterRegex = /alter\s+table\s+(?:only\s+)?(?:public\.)?(\w+)/gi; + const tables = new Set(); + + let match; + while ((match = tableRegex.exec(sql)) !== null) { + tables.add(match[1].toLowerCase()); + } + while ((match = alterRegex.exec(sql)) !== null) { + tables.add(match[1].toLowerCase()); + } + + return tables; +} + +function extractIndexes(sql: string): Set { + const indexRegex = /create\s+(?:unique\s+)?index\s+(?:if\s+not\s+exists\s+)?(\w+)/gi; + const indexes = new Set(); + + let match; + while ((match = indexRegex.exec(sql)) !== null) { + indexes.add(match[1].toLowerCase()); + } + + return indexes; +} + +function extractPolicies(sql: string): Set { + const policyRegex = /create\s+policy\s+(\w+)/gi; + const policies = new Set(); + + let match; + while ((match = policyRegex.exec(sql)) !== null) { + policies.add(match[1].toLowerCase()); + } + + return policies; +} + +interface MigrationData { + content: string; + normalized: string; + tables: Set; + indexes: Set; + policies: Set; +} + +function readMigrations(dir: string): Map { + const migrations = new Map(); + + if (!fs.existsSync(dir)) { + return migrations; + } + + const files = fs.readdirSync(dir).filter(f => f.endsWith('.sql')).sort(); + + for (const file of files) { + const filePath = path.join(dir, file); + const content = fs.readFileSync(filePath, 'utf-8'); + migrations.set(file, { + content, + normalized: normalizeSQL(content), + tables: extractTables(content), + indexes: extractIndexes(content), + policies: extractPolicies(content) + }); + } + + return migrations; +} + +interface Issue { + type: string; + severity: 'error' | 'warning'; + message: string; + files?: string[]; + tables?: string[]; + migration?: string; + executedAt?: string; +} + +function compareMigrations(name1: string, m1: MigrationData, name2: string, m2: MigrationData): Issue[] { + const issues: Issue[] = []; + + // Check if normalized content is identical + if (m1.normalized === m2.normalized) { + issues.push({ + type: 'duplicate', + severity: 'error', + message: `Identical migrations: "${name1}" and "${name2}"`, + files: [name1, name2] + }); + } else { + // Check for table overlap with different content + const commonTables = [...m1.tables].filter(t => m2.tables.has(t)); + if (commonTables.length > 0) { + issues.push({ + type: 'conflict', + severity: 'warning', + message: `Common tables in different migrations: "${name1}" and "${name2}" affect tables: ${commonTables.join(', ')}`, + files: [name1, name2], + tables: commonTables + }); + } + } + + return issues; +} + +describe('Migration Drift Check', () => { + describe('SQL Normalization', () => { + it('should remove single-line comments', () => { + const sql = ` + CREATE TABLE users (id INT); -- this is a comment + INSERT INTO users VALUES (1); -- another comment + `; + const normalized = normalizeSQL(sql); + expect(normalized).not.toContain('--'); + expect(normalized).toContain('create table users'); + }); + + it('should remove multi-line comments', () => { + const sql = ` + /* This is a + multi-line comment */ + CREATE TABLE users (id INT); + `; + const normalized = normalizeSQL(sql); + expect(normalized).not.toContain('/*'); + expect(normalized).not.toContain('*/'); + expect(normalized).toContain('create table users'); + }); + + it('should normalize whitespace', () => { + const sql = `CREATE TABLE users ( id INT )`; + const normalized = normalizeSQL(sql); + expect(normalized).toBe('create table users ( id int )'); + }); + + it('should be case-insensitive', () => { + const sql1 = 'CREATE TABLE Users (ID INT)'; + const sql2 = 'create table users (id int)'; + expect(normalizeSQL(sql1)).toBe(normalizeSQL(sql2)); + }); + + it('should handle identical migrations with different formatting', () => { + const sql1 = ` + -- Create users table + CREATE TABLE public.users ( + id BIGINT PRIMARY KEY + ); + `; + const sql2 = `CREATE TABLE users(id BIGINT PRIMARY KEY);`; + expect(normalizeSQL(sql1)).toBe(normalizeSQL(sql2)); + }); + }); + + describe('Table Extraction', () => { + it('should extract table names from CREATE TABLE statements', () => { + const sql = 'CREATE TABLE users (id INT); CREATE TABLE orders (id INT);'; + const tables = extractTables(sql); + expect(tables).toContain('users'); + expect(tables).toContain('orders'); + expect(tables.size).toBe(2); + }); + + it('should handle IF NOT EXISTS clause', () => { + const sql = 'CREATE TABLE IF NOT EXISTS users (id INT);'; + const tables = extractTables(sql); + expect(tables).toContain('users'); + }); + + it('should handle public schema prefix', () => { + const sql = 'CREATE TABLE public.users (id INT);'; + const tables = extractTables(sql); + expect(tables).toContain('users'); + }); + + it('should extract tables from ALTER TABLE statements', () => { + const sql = 'ALTER TABLE users ADD COLUMN email VARCHAR(255);'; + const tables = extractTables(sql); + expect(tables).toContain('users'); + }); + + it('should extract multiple tables correctly', () => { + const sql = ` + CREATE TABLE users (id INT); + ALTER TABLE users ADD COLUMN name VARCHAR(100); + CREATE TABLE products (id INT); + `; + const tables = extractTables(sql); + expect(tables).toContain('users'); + expect(tables).toContain('products'); + expect(tables.size).toBe(2); + }); + }); + + describe('Index Extraction', () => { + it('should extract index names', () => { + const sql = 'CREATE INDEX idx_users_email ON users(email);'; + const indexes = extractIndexes(sql); + expect(indexes).toContain('idx_users_email'); + }); + + it('should handle UNIQUE INDEX', () => { + const sql = 'CREATE UNIQUE INDEX idx_users_email ON users(email);'; + const indexes = extractIndexes(sql); + expect(indexes).toContain('idx_users_email'); + }); + + it('should handle IF NOT EXISTS', () => { + const sql = 'CREATE INDEX IF NOT EXISTS idx_users_id ON users(id);'; + const indexes = extractIndexes(sql); + expect(indexes).toContain('idx_users_id'); + }); + }); + + describe('Policy Extraction', () => { + it('should extract policy names', () => { + const sql = 'CREATE POLICY user_policy ON users FOR SELECT USING (auth.uid() = id);'; + const policies = extractPolicies(sql); + expect(policies).toContain('user_policy'); + }); + + it('should extract multiple policies', () => { + const sql = ` + CREATE POLICY select_policy ON users FOR SELECT USING (true); + CREATE POLICY update_policy ON users FOR UPDATE USING (auth.uid() = id); + `; + const policies = extractPolicies(sql); + expect(policies).toContain('select_policy'); + expect(policies).toContain('update_policy'); + }); + }); + + describe('Migration Comparison', () => { + it('should detect identical migrations', () => { + const m1: MigrationData = { + content: 'CREATE TABLE users (id INT);', + normalized: 'create table users (id int);', + tables: new Set(['users']), + indexes: new Set(), + policies: new Set() + }; + const m2: MigrationData = { + content: 'CREATE TABLE users (id INT);', + normalized: 'create table users (id int);', + tables: new Set(['users']), + indexes: new Set(), + policies: new Set() + }; + + const issues = compareMigrations('001_create_users.sql', m1, '001_create_users.sql', m2); + expect(issues).toHaveLength(1); + expect(issues[0].type).toBe('duplicate'); + expect(issues[0].severity).toBe('error'); + }); + + it('should detect table conflicts between different migrations', () => { + const m1: MigrationData = { + content: 'CREATE TABLE users (id INT);', + normalized: 'create table users (id int);', + tables: new Set(['users']), + indexes: new Set(), + policies: new Set() + }; + const m2: MigrationData = { + content: 'ALTER TABLE users ADD COLUMN email VARCHAR(255);', + normalized: 'alter table users add column email varchar(255);', + tables: new Set(['users']), + indexes: new Set(), + policies: new Set() + }; + + const issues = compareMigrations('001_create_users.sql', m1, '002_alter_users.sql', m2); + expect(issues).toHaveLength(1); + expect(issues[0].type).toBe('conflict'); + expect(issues[0].severity).toBe('warning'); + expect(issues[0].tables).toContain('users'); + }); + + it('should not flag migrations affecting different tables', () => { + const m1: MigrationData = { + content: 'CREATE TABLE users (id INT);', + normalized: 'create table users (id int);', + tables: new Set(['users']), + indexes: new Set(), + policies: new Set() + }; + const m2: MigrationData = { + content: 'CREATE TABLE products (id INT);', + normalized: 'create table products (id int);', + tables: new Set(['products']), + indexes: new Set(), + policies: new Set() + }; + + const issues = compareMigrations('001_create_users.sql', m1, '002_create_products.sql', m2); + expect(issues).toHaveLength(0); + }); + }); + + describe('Migration File Reading', () => { + let tempDir: string; + + beforeEach(() => { + tempDir = path.join(__dirname, '.test-migrations-' + Date.now()); + fs.mkdirSync(tempDir, { recursive: true }); + }); + + afterEach(() => { + if (fs.existsSync(tempDir)) { + fs.rmSync(tempDir, { recursive: true }); + } + }); + + it('should read SQL migration files', () => { + fs.writeFileSync(path.join(tempDir, '001_init.sql'), 'CREATE TABLE users (id INT);'); + fs.writeFileSync(path.join(tempDir, '002_add_email.sql'), 'ALTER TABLE users ADD COLUMN email VARCHAR(255);'); + + const migrations = readMigrations(tempDir); + expect(migrations.size).toBe(2); + expect(migrations.has('001_init.sql')).toBe(true); + expect(migrations.has('002_add_email.sql')).toBe(true); + }); + + it('should ignore non-SQL files', () => { + fs.writeFileSync(path.join(tempDir, '001_init.sql'), 'CREATE TABLE users (id INT);'); + fs.writeFileSync(path.join(tempDir, 'README.md'), '# Migrations'); + fs.writeFileSync(path.join(tempDir, '.gitkeep'), ''); + + const migrations = readMigrations(tempDir); + expect(migrations.size).toBe(1); + expect(migrations.has('001_init.sql')).toBe(true); + }); + + it('should handle non-existent directory gracefully', () => { + const nonExistent = path.join(tempDir, 'nonexistent'); + const migrations = readMigrations(nonExistent); + expect(migrations.size).toBe(0); + }); + + it('should parse migration metadata correctly', () => { + const sql = 'CREATE TABLE users (id INT); CREATE INDEX idx_users_id ON users(id);'; + fs.writeFileSync(path.join(tempDir, '001_init.sql'), sql); + + const migrations = readMigrations(tempDir); + const migration = migrations.get('001_init.sql')!; + + expect(migration.content).toBe(sql); + expect(migration.normalized).toContain('create table users'); + expect(migration.tables).toContain('users'); + expect(migration.indexes).toContain('idx_users_id'); + }); + }); + + describe('Filesystem Migration Analysis', () => { + it('should detect migrations in actual filesystem', () => { + const backendMigrations = readMigrations(BACKEND_MIGRATIONS); + const supabaseMigrations = readMigrations(SUPABASE_MIGRATIONS); + + // Check that we have migrations + expect(backendMigrations.size).toBeGreaterThan(0); + expect(supabaseMigrations.size).toBeGreaterThan(0); + + // All should have parsed metadata + for (const migration of backendMigrations.values()) { + expect(migration.content).toBeTruthy(); + expect(migration.normalized).toBeTruthy(); + expect(migration.tables).toBeTruthy(); + } + }); + + it('should identify migration naming patterns', () => { + const backendMigrations = readMigrations(BACKEND_MIGRATIONS); + const names = Array.from(backendMigrations.keys()); + + // Check for various naming patterns + const hasTimestampPattern = names.some(n => /^\d{14}/.test(n)); + const hasSequentialPattern = names.some(n => /^\d{3}_/.test(n)); + + expect(hasTimestampPattern || hasSequentialPattern).toBe(true); + }); + }); + + describe('Database State Validation', () => { + it('should identify unapplied migrations', () => { + const appliedNames = new Set(['001_init.sql', '002_users.sql']); + const filesystemMigrations = new Set(['001_init.sql', '002_users.sql', '003_products.sql']); + + const unapplied: string[] = []; + for (const file of filesystemMigrations) { + if (!appliedNames.has(file)) { + unapplied.push(file); + } + } + + expect(unapplied).toEqual(['003_products.sql']); + }); + + it('should identify orphaned migrations', () => { + const appliedNames = ['001_init.sql', '002_users.sql', '003_old_removed.sql']; + const filesystemMigrations = new Set(['001_init.sql', '002_users.sql']); + + const orphaned: string[] = []; + for (const name of appliedNames) { + if (!filesystemMigrations.has(name)) { + orphaned.push(name); + } + } + + expect(orphaned).toEqual(['003_old_removed.sql']); + }); + + it('should compare database and filesystem states', () => { + const applied = [ + { name: '001_init.sql', executedAt: '2024-01-01T00:00:00Z' }, + { name: '002_users.sql', executedAt: '2024-01-02T00:00:00Z' } + ]; + const filesystem = new Set(['001_init.sql', '002_users.sql', '003_products.sql']); + + const issues: Issue[] = []; + const appliedSet = new Set(applied.map(m => m.name)); + + for (const file of filesystem) { + if (!appliedSet.has(file)) { + issues.push({ + type: 'unapplied_migration', + severity: 'warning', + migration: file, + message: `Migration in filesystem but not applied to database: "${file}"` + }); + } + } + + for (const m of applied) { + if (!filesystem.has(m.name)) { + issues.push({ + type: 'orphaned_migration', + severity: 'warning', + migration: m.name, + message: `Migration in database but not in filesystem: "${m.name}"`, + executedAt: m.executedAt + }); + } + } + + expect(issues).toHaveLength(1); + expect(issues[0].type).toBe('unapplied_migration'); + expect(issues[0].migration).toBe('003_products.sql'); + }); + }); + + describe('Error Reporting', () => { + it('should format error messages clearly', () => { + const issue: Issue = { + type: 'duplicate', + severity: 'error', + message: 'Identical migrations: "001_init.sql" and "001_init_backup.sql"', + files: ['001_init.sql', '001_init_backup.sql'] + }; + + expect(issue.message).toContain('Identical migrations'); + expect(issue.message).toContain('001_init.sql'); + expect(issue.files).toHaveLength(2); + }); + + it('should separate errors from warnings', () => { + const issues: Issue[] = [ + { type: 'duplicate', severity: 'error', message: 'Duplicate found' }, + { type: 'conflict', severity: 'warning', message: 'Conflict warning' }, + { type: 'duplicate', severity: 'error', message: 'Another duplicate' } + ]; + + const errors = issues.filter(i => i.severity === 'error'); + const warnings = issues.filter(i => i.severity === 'warning'); + + expect(errors).toHaveLength(2); + expect(warnings).toHaveLength(1); + }); + }); + + describe('Environment-Aware Output', () => { + it('should include env field in JSON output structure', () => { + const output = { + success: true, + env: 'local', + fileCheck: { backendCount: 5, supabaseCount: 5 }, + dbCheck: null, + issues: [] + }; + + expect(output.env).toBe('local'); + expect(output).toHaveProperty('env'); + }); + + it('should distinguish local from ci-remote environment labels', () => { + const localOutput = { env: 'local', success: true, issues: [] }; + const ciOutput = { env: 'ci-remote', success: true, issues: [] }; + + expect(localOutput.env).not.toBe(ciOutput.env); + expect(localOutput.env).toBe('local'); + expect(ciOutput.env).toBe('ci-remote'); + }); + + it('should include env label in issue context for cross-environment comparison', () => { + const localResult = { + env: 'local', + dbCheck: { + appliedCount: 10, + filesystemCount: 12, + issues: [ + { + type: 'unapplied_migration', + severity: 'warning' as const, + message: 'Migration in filesystem but not applied to database: "20260530_new_index.sql"', + migration: '20260530_new_index.sql' + } + ] + } + }; + const ciResult = { + env: 'ci-remote', + dbCheck: { + appliedCount: 12, + filesystemCount: 12, + issues: [] + } + }; + + // Local has unapplied migrations that CI doesn't ā€” this is a cross-environment divergence + expect(localResult.dbCheck.issues).toHaveLength(1); + expect(ciResult.dbCheck.issues).toHaveLength(0); + expect(localResult.dbCheck.appliedCount).toBeLessThan(ciResult.dbCheck.appliedCount); + expect(localResult.env).toBe('local'); + expect(ciResult.env).toBe('ci-remote'); + }); + }); + + describe('Conflicting Migration History Detection', () => { + it('should identify migrations present in CI database but not local', () => { + const localApplied = new Set(['001_init.sql', '002_users.sql']); + const ciApplied = new Set(['001_init.sql', '002_users.sql', '003_products.sql']); + + const onlyInCI: string[] = []; + for (const name of ciApplied) { + if (!localApplied.has(name)) { + onlyInCI.push(name); + } + } + + expect(onlyInCI).toEqual(['003_products.sql']); + }); + + it('should identify migrations present locally but not in CI', () => { + const localApplied = new Set(['001_init.sql', '002_users.sql', '003_local_only.sql']); + const ciApplied = new Set(['001_init.sql', '002_users.sql']); + + const onlyInLocal: string[] = []; + for (const name of localApplied) { + if (!ciApplied.has(name)) { + onlyInLocal.push(name); + } + } + + expect(onlyInLocal).toEqual(['003_local_only.sql']); + }); + + it('should detect when applied migration count diverges between environments', () => { + const localCount = 10; + const ciCount = 12; + + expect(localCount).not.toBe(ciCount); + const diverged = Math.abs(localCount - ciCount) > 0; + expect(diverged).toBe(true); + }); + + it('should surface conflicting history with executedAt timestamps', () => { + const orphanedIssue: Issue = { + type: 'orphaned_migration', + severity: 'warning', + message: 'Migration in database but not in filesystem: "20260101_dropped.sql"', + migration: '20260101_dropped.sql', + executedAt: '2026-01-01T10:00:00Z' + }; + + expect(orphanedIssue.executedAt).toBeDefined(); + expect(orphanedIssue.message).toContain('20260101_dropped.sql'); + expect(orphanedIssue.type).toBe('orphaned_migration'); + }); + }); +}); diff --git a/docs/MIGRATION_DRIFT_CHECKING.md b/docs/MIGRATION_DRIFT_CHECKING.md new file mode 100644 index 00000000..d9cb123c --- /dev/null +++ b/docs/MIGRATION_DRIFT_CHECKING.md @@ -0,0 +1,318 @@ +# Migration Drift Checking + +## Overview + +This document describes the enhanced migration drift checking system for the SYNCRO repository. It helps detect and prevent divergence between the `backend/migrations` and `supabase/migrations` directories and validates that applied migrations match the filesystem. + +## Quick Start + +### File-Only Drift Check (Fastest) +```bash +npm run check:migrations +``` + +### Full Verification (File + Database) +```bash +npm run check:migrations --verify-db +``` + +### JSON Output (for CI/tooling) +```bash +npm run check:migrations --json +npm run check:migrations --verify-db --json +``` + +## Tools + +### 1. Enhanced Drift Check Script (`scripts/check-migration-drift.js`) + +Performs static file analysis and optional database verification. + +**Modes:** +- **File-only** (default): Compares migration files, detects duplicates and conflicts +- **Database verification** (`--verify-db`): Also checks that applied migrations match filesystem + +**Features:** +- Normalizes SQL (removes comments, whitespace) for accurate comparison +- Extracts and compares table, index, and policy names +- Identifies duplicate migrations (identical SQL in different files) +- Identifies conflicts (different migrations affecting the same tables) +- Optional database state verification + +**Output Formats:** +- Human-readable (default) +- JSON (`--json`) + +**Exit Codes:** +- `0`: No drift detected +- `1`: Drift detected (errors found) +- `2`: Error occurred (connection, syntax, etc.) + +**Example Usage:** +```bash +# File check only +node scripts/check-migration-drift.js + +# With database verification +SUPABASE_URL=... SUPABASE_SERVICE_ROLE_KEY=... node scripts/check-migration-drift.js --verify-db + +# JSON output +node scripts/check-migration-drift.js --json > drift-report.json +``` + +### 2. Migration State Validator (`scripts/validate-migration-state.js`) + +Standalone utility to inspect the actual database migration state. Useful for: +- Comparing applied migrations across environments +- Debugging unapplied or orphaned migrations +- Validating database consistency + +**Features:** +- Fetches applied migration history from database +- Lists database tables +- Compares filesystem migrations with applied state +- Supports multiple environments + +**Output Formats:** +- Human-readable (default) +- JSON (`--json`) + +**Exit Codes:** +- `0`: Validation successful +- `1`: Issues found +- `2`: Error occurred + +**Example Usage:** +```bash +# Check current database state +SUPABASE_URL=... SUPABASE_SERVICE_ROLE_KEY=... node scripts/validate-migration-state.js + +# Compare with filesystem +SUPABASE_URL=... SUPABASE_SERVICE_ROLE_KEY=... node scripts/validate-migration-state.js --compare-files + +# JSON output for parsing +node scripts/validate-migration-state.js --json | jq '.appliedMigrations' +``` + +## CI/CD Integration + +### GitHub Actions Workflow + +The workflow in `.github/workflows/migration-drift-check.yml` includes: + +1. **File-level drift check** (always runs) + - Compares `backend/migrations/` and `supabase/migrations/` + - Posts PR comment with results + - Blocks merge if duplicates found + +2. **Database verification** (optional, requires secrets) + - Verifies applied migrations against filesystem + - Only runs on main branch or if `ENABLE_DB_VERIFICATION=true` repo variable + - Requires `SUPABASE_URL` and `SUPABASE_SERVICE_ROLE_KEY` secrets + +### Enabling Database Verification in CI + +1. Set repository variable: `ENABLE_DB_VERIFICATION` = `true` +2. Ensure secrets are configured: + - `SUPABASE_URL`: Your Supabase project URL + - `SUPABASE_SERVICE_ROLE_KEY`: Service role key (for admin access) + +The database check will then run on every PR and main branch push that modifies migrations. + +## Types of Drift + +### Duplicate Migrations (ERROR) +Two migration files contain identical SQL. + +**Example:** +``` +āŒ ERRORS (must fix): + [DUPLICATE] Identical migrations: "001_init.sql" and "001_init_backup.sql" +``` + +**Fix:** Remove one of the files, keep the canonical version. +See [MIGRATION_REMEDIATION.md](./MIGRATION_REMEDIATION.md#case-1-duplicate-migrations) + +### Conflict Migrations (WARNING) +Different migrations modify the same table(s). + +**Example:** +``` +āš ļø WARNINGS (review recommended): + [CONFLICT] Common tables in different migrations: "001_create_users.sql" and + "002_alter_users.sql" affect tables: users +``` + +**Fix:** Consolidate migrations to a single source. See [MIGRATION_REMEDIATION.md](./MIGRATION_REMEDIATION.md#case-2-conflict-migrations) + +### Unapplied Migrations (WARNING - Database Check) +A migration exists in the filesystem but hasn't been applied to the database. + +**Example:** +``` +=== Database Verification Issues === + [UNAPPLIED_MIGRATION] Migration in filesystem but not applied to database: "20260529_new_feature.sql" +``` + +**Fix:** Run `npm run db:migrate` to apply pending migrations. +See [MIGRATION_REMEDIATION.md](./MIGRATION_REMEDIATION.md#case-3-unapplied-migrations-local-development) + +### Orphaned Migrations (WARNING - Database Check) +A migration has been applied to the database but doesn't exist in the filesystem. + +**Example:** +``` +=== Database Verification Issues === + [ORPHANED_MIGRATION] Migration in database but not in filesystem: "20260101_old_removed.sql" +``` + +**Fix:** Either restore the migration file or clean up the database history. +See [MIGRATION_REMEDIATION.md](./MIGRATION_REMEDIATION.md#case-5-orphaned-migrations-database-has-migration-filesystem-doesnt) + +## Understanding the Output + +### File-Only Check Output +``` +šŸ” Checking migration drift between backend and supabase... + +=== Migration Analysis === + +Backend migrations: 29 files +Supabase migrations: 28 files + +āœ… No migration drift detected. + +--- Summary --- +Total backend tables: 45 +Total supabase tables: 47 +``` + +### Database Verification Output +``` +=== Database Verification Issues === + + [UNAPPLIED_MIGRATION] Migration in filesystem but not applied to database: "20260529_new_feature.sql" + [ORPHANED_MIGRATION] Migration in database but not in filesystem: "20260101_removed.sql" +``` + +### JSON Output Format +```json +{ + "success": true, + "fileCheck": { + "backendCount": 29, + "supabaseCount": 28, + "backendTableCount": 45, + "supabaseTableCount": 47, + "errors": 0, + "warnings": 0 + }, + "dbCheck": { + "appliedCount": 57, + "filesystemCount": 57, + "issues": [], + "appliedMigrations": [ + { + "name": "20240101000000_create_audit_logs.sql", + "executedAt": "2024-01-01T00:00:00Z" + } + ] + }, + "issues": [] +} +``` + +## Best Practices + +### 1. Use a Single Source of Truth +Choose **one** authoritative location for migrations: +- Option A (Recommended): All migrations in `supabase/migrations/` +- Option B: All migrations in `backend/migrations/` + +Never maintain parallel migration trees. + +### 2. Check Before Committing +```bash +npm run check:migrations +``` + +### 3. Check Before Pushing +```bash +npm run check:migrations --verify-db +``` + +### 4. Review Migration Files in PRs +```bash +git diff HEAD~1..HEAD -- backend/migrations/ supabase/migrations/ +``` + +### 5. Apply Migrations Locally +```bash +npm run db:migrate +npm run check:migrations --verify-db +``` + +### 6. Monitor CI Results +- Read drift check comments on PRs +- Fix issues before merging +- Don't ignore warnings + +## Troubleshooting + +### "Database connection failed" +```bash +# Verify environment variables +echo $SUPABASE_URL +echo $SUPABASE_SERVICE_ROLE_KEY + +# Test connection +curl -X POST "$SUPABASE_URL/rest/v1/rpc/exec_sql" \ + -H "Authorization: Bearer $SUPABASE_SERVICE_ROLE_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "SELECT 1"}' +``` + +### "Migration syntax error" +```bash +# Validate SQL +sqlcheck < supabase/migrations/your_migration.sql + +# Test with psql +psql "$DATABASE_URL" < supabase/migrations/your_migration.sql +``` + +### "Too many conflicts to resolve" +See [MIGRATION_REMEDIATION.md](./MIGRATION_REMEDIATION.md#comprehensive-troubleshooting) + +## Related Documentation + +- [MIGRATION_REMEDIATION.md](./MIGRATION_REMEDIATION.md) ā€” Detailed fix procedures +- [backend/README.md](../backend/README.md) ā€” Backend setup +- [supabase/README.md](../supabase/README.md) ā€” Supabase setup + +## Environment Variables + +For database verification (`--verify-db`): + +| Variable | Required | Description | +|----------|----------|-------------| +| `SUPABASE_URL` | Yes | Supabase project URL | +| `SUPABASE_SERVICE_ROLE_KEY` | Yes | Service role key (admin access) | +| `DATABASE_URL` | No | Direct Postgres URL (overrides Supabase) | + +## Contributing + +When adding new migrations: + +1. Create the migration file in the canonical location (e.g., `supabase/migrations/`) +2. Write clear, idempotent SQL +3. Add comments explaining the change +4. Run local tests: `npm run check:migrations --verify-db` +5. Open PR and let CI verify +6. Address any drift issues before merging + +## Support + +For issues not covered here, see: +- [docs/MIGRATION_REMEDIATION.md](./MIGRATION_REMEDIATION.md) +- Issue tracker: Create a new issue with drift check output diff --git a/docs/MIGRATION_DRIFT_QUICK_REFERENCE.md b/docs/MIGRATION_DRIFT_QUICK_REFERENCE.md new file mode 100644 index 00000000..16fa77e6 --- /dev/null +++ b/docs/MIGRATION_DRIFT_QUICK_REFERENCE.md @@ -0,0 +1,113 @@ +# Migration Drift Checks ā€” Quick Reference + +## Common Commands + +```bash +# Check for migration drift (file-only) +npm run check:migrations + +# Check with database verification +npm run check:migrations:verify-db + +# Inspect database migration state +npm run validate:migration-state + +# Compare with filesystem +npm run validate:migration-state --compare-files + +# Get JSON output (for tooling) +npm run check:migrations --json + +# Run migration tests +npm run test:migrations +``` + +## Issue Types & Fixes + +| Issue | Severity | Fix | +|-------|----------|-----| +| **DUPLICATE** | ERROR | Remove one copy, keep canonical file | +| **CONFLICT** | WARNING | Consolidate to single source (supabase/migrations/) | +| **UNAPPLIED_MIGRATION** | WARNING | Run `npm run db:migrate` | +| **ORPHANED_MIGRATION** | WARNING | Restore file or clean database history | + +## Before Committing + +```bash +npm run check:migrations +# Fix any errors before committing +``` + +## Before Pushing + +```bash +npm run check:migrations:verify-db +# Fix any issues, ensure database is in sync +``` + +## CI Will + +1. āœ… Run file check on every PR with migration changes +2. šŸ“Š Run optional database check on main branch +3. šŸ’¬ Post results as PR comments +4. šŸ”— Link to remediation guide if issues found + +## Remediation Guides + +- **All issue types:** `docs/MIGRATION_REMEDIATION.md` +- **Feature overview:** `docs/MIGRATION_DRIFT_CHECKING.md` + +## JSON Output + +```bash +npm run check:migrations --json +# Output includes: +# - fileCheck: file-level analysis results +# - dbCheck: database verification results (if --verify-db) +# - issues: detailed list of found issues +# - success: overall pass/fail status +``` + +## Environment Setup + +For database checks (`--verify-db`): + +```bash +export SUPABASE_URL="https://your-project.supabase.co" +export SUPABASE_SERVICE_ROLE_KEY="your-service-role-key" +npm run check:migrations:verify-db +``` + +## Emergency Reference + +**Everything working?** +```bash +npm run check:migrations +# Should output: āœ… No migration drift detected. +``` + +**Something broken?** +1. Read the error message +2. Find your issue type in `docs/MIGRATION_REMEDIATION.md` +3. Follow the step-by-step fix +4. Verify with `npm run check:migrations:verify-db` + +**Need more info?** +- `docs/MIGRATION_DRIFT_CHECKING.md` ā€” Full feature guide +- `docs/MIGRATION_REMEDIATION.md` ā€” Detailed fixes +- Test cases in `backend/tests/migration-drift.test.ts` ā€” Example scenarios + +## Troubleshooting + +| Problem | Solution | +|---------|----------| +| "Database connection failed" | Set SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY | +| "Migration syntax error" | Check SQL syntax: `sqlcheck < migration.sql` | +| "Too many conflicts" | See MIGRATION_REMEDIATION.md ā†’ Comprehensive Troubleshooting | +| "Tests failing" | Run `npm run test:migrations` for details | + +--- + +**Last Updated:** Issue #657 Implementation + +For detailed information, see the full documentation in `docs/`. diff --git a/docs/MIGRATION_REMEDIATION.md b/docs/MIGRATION_REMEDIATION.md new file mode 100644 index 00000000..857baec1 --- /dev/null +++ b/docs/MIGRATION_REMEDIATION.md @@ -0,0 +1,434 @@ +# Migration Remediation Guide + +## Overview + +This guide provides step-by-step instructions for remediating migration drift issues detected by the migration drift check tools. + +## Understanding Issue Types + +### Duplicate Migrations +**Severity:** ERROR ā€” Must be fixed before merging + +Two migration files contain identical SQL. This usually indicates: +- A migration was copied by mistake +- The same migration was added to both `backend/migrations` and `supabase/migrations` + +### Conflict Migrations +**Severity:** WARNING ā€” Review before merging + +Two different migrations modify the same table(s). While not always an error, it indicates: +- Schema changes are split across multiple locations +- Potential for state divergence between `backend/migrations` and `supabase/migrations` + +### Unapplied Migrations +**Severity:** WARNING ā€” Review database state + +A migration exists in the filesystem but hasn't been applied to the database. This can occur: +- After pulling new migration files from git +- If a migration failed to apply in a previous deploy +- During local development before running `npm run db:migrate` + +### Orphaned Migrations +**Severity:** WARNING ā€” Clean up or recover + +A migration has been applied to the database but doesn't exist in the filesystem. This usually means: +- A migration file was accidentally deleted +- The migration was added directly to the database without a corresponding file +- A rollback wasn't properly tracked + +--- + +## Remediation Procedures + +### Case 1: Duplicate Migrations + +**Issue:** `Identical migrations: "001_init.sql" and "001_init_backup.sql"` + +#### Step 1: Identify which copy is correct +```bash +# List all migration files +ls backend/migrations/ +ls supabase/migrations/ + +# Compare the two files +diff backend/migrations/001_init.sql supabase/migrations/001_init.sql +``` + +#### Step 2: Keep the correct version +- If they're truly identical, keep the one in **`supabase/migrations/`** (canonical location) +- If they're different, keep both but rename to avoid duplication + +#### Step 3: Remove the duplicate +```bash +# Remove from backend/migrations if it's truly identical +rm backend/migrations/001_init.sql + +# Or remove from supabase/migrations (less common) +rm supabase/migrations/001_init.sql + +git add -A +git commit -m "Remove duplicate migration 001_init.sql" +``` + +#### Step 4: Verify the fix +```bash +npm run check:migrations # File check passes +npm run check:migrations --verify-db # Database check passes +``` + +--- + +### Case 2: Conflict Migrations + +**Issue:** `Common tables in different migrations: "001_create_users.sql" and "002_alter_users.sql" affect tables: users` + +This often indicates that schema changes are split across incompatible migration paths. + +#### Step 1: Understand the sequence +```bash +# View the content of both migrations +cat backend/migrations/001_create_users.sql +cat supabase/migrations/002_alter_users.sql +``` + +#### Step 2: Consolidate into a single path +Choose one authoritative location (`supabase/migrations/` is recommended): + +```bash +# Option A: Copy all migrations to supabase/migrations, remove from backend +cp backend/migrations/*.sql supabase/migrations/ +rm backend/migrations/*.sql + +git add -A +git commit -m "Consolidate all migrations to supabase/migrations/" +``` + +Or: + +```bash +# Option B: Keep backend/migrations, remove from supabase (less preferred) +cp supabase/migrations/*.sql backend/migrations/ +rm supabase/migrations/*.sql + +git add -A +git commit -m "Consolidate all migrations to backend/migrations/" +``` + +#### Step 3: Verify +```bash +npm run check:migrations +``` + +--- + +### Case 3: Unapplied Migrations (Local Development) + +**Issue:** `Migration in filesystem but not applied to database: "20260529_new_feature.sql"` + +This is common after pulling from git or creating new migrations. + +#### Step 1: Apply the migration +```bash +# Push migrations to your local database +npm run db:migrate + +# Or push to a specific database +SUPABASE_URL= SUPABASE_SERVICE_ROLE_KEY= npm run db:migrate +``` + +#### Step 2: Verify +```bash +npm run check:migrations --verify-db +``` + +If the issue persists after `db:migrate`, the migration may have failed. Check logs: + +```bash +# View Supabase logs +supabase logs --follow +``` + +--- + +### Case 4: Unapplied Migrations (CI Database) + +**Issue:** Migration check fails in CI with unapplied migrations + +#### Step 1: Check the CI database state +```bash +# Use the validation script to inspect the CI database +DATABASE_URL= node scripts/validate-migration-state.js --json +``` + +#### Step 2: Apply missing migrations +If the CI database is behind: + +```bash +# Option A: Run migrations in CI pipeline (recommended) +# Update .github/workflows/migration-drift-check.yml to run migrations: +- name: Apply pending migrations + run: npm run db:migrate + env: + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} +``` + +Or: + +```bash +# Option B: Manually trigger migration in CI database (one-time fix) +npx supabase db push \ + --db-url "$CI_DATABASE_URL" \ + --password "$CI_DB_PASSWORD" +``` + +#### Step 3: Verify CI passes +Once migrations are applied, re-run CI checks. + +--- + +### Case 5: Orphaned Migrations (Database has migration, filesystem doesn't) + +**Issue:** `Migration in database but not in filesystem: "20260101_old_removed.sql"` + +This indicates a migration was deleted from version control but remains in the database. + +#### Step 1: Decide whether to restore or ignore + +If the migration is **critical** (e.g., created a core table): +```bash +# Restore the migration file from git history +git log --all --pretty=format:"%H %s" | grep -i "20260101" + +# Check out that version +git show :supabase/migrations/20260101_old_removed.sql > \ + supabase/migrations/20260101_old_removed.sql + +git add supabase/migrations/20260101_old_removed.sql +git commit -m "Restore deleted migration 20260101_old_removed.sql" +``` + +If the migration is **safe to remove** (e.g., housekeeping, temporary test data): +```bash +# Manually delete from database (CAREFUL ā€” requires admin access) +SUPABASE_URL= SUPABASE_SERVICE_ROLE_KEY= node scripts/validate-migration-state.js +# Record the orphaned migration name +# Then remove it from supabase_migrations_history table via dashboard +``` + +#### Step 2: Verify +```bash +npm run check:migrations --verify-db +``` + +--- + +### Case 6: Cross-Environment Divergence + +**Issue:** Local database applied migrations that CI database doesn't have (or vice versa) + +CI automatically detects this via two parallel checks on every PR: +- **`verify-local-database` job** ā€” spins up a fresh local Supabase stack and checks filesystem vs. applied state +- **`verify-database` job** ā€” checks the CI remote database (requires `ENABLE_DB_VERIFICATION=true` on PRs, or runs automatically on pushes to `main`) + +When these jobs produce different `appliedCount` values, it signals cross-environment divergence. + +#### Step 1: Read the CI output + +PR comments will show, per environment: + +``` +Applied: 10 | Filesystem: 12 +[unapplied_migration] Migration in filesystem but not applied to database: "20260530_new_index.sql" +``` + +#### Step 2: Compare environments locally +```bash +# Check local environment +node scripts/check-migration-drift.js --verify-db --env local --json + +# Check CI remote environment (if you have credentials) +SUPABASE_URL= SUPABASE_SERVICE_ROLE_KEY= \ + node scripts/check-migration-drift.js --verify-db --env ci-remote --json + +# Validate raw migration state in either environment +node scripts/validate-migration-state.js --json +``` + +#### Step 3: Align the databases + +If **local is behind CI** (CI has more applied migrations): +```bash +# Pull the latest migrations from git +git pull origin main + +# Apply them locally +npm run db:migrate +``` + +If **local is ahead of CI** (local applied migrations CI hasn't seen): +```bash +# Don't merge the PR until CI catches up. +# Push the pending migrations to the CI database: +npm run db:migrate:prod --db-url "$CI_DATABASE_URL" +``` + +If **both environments are missing the same migration** (filesystem has it, neither DB has it): +```bash +# Apply to local +npm run db:migrate + +# CI will auto-apply on next pipeline run +``` + +#### Step 4: Verify both environments match +```bash +node scripts/check-migration-drift.js --verify-db --env local --json +``` + +--- + +### Case 7: CI Drift Check Finds Issues Not Present Locally + +**Issue:** `verify-local-database` CI job passes but `verify-database` (CI remote) fails with orphaned or unapplied migrations + +This means the CI remote database has diverged from the filesystem ā€” likely because: +- A migration was applied to the CI database manually (outside of version control) +- A migration file was deleted from git after being deployed +- The CI database has never been reset and accumulated ad-hoc changes + +#### Step 1: Identify the discrepancy +```bash +# From CI logs, note the orphaned migrations reported by verify-database job +# Example: [orphaned_migration] Migration in database but not in filesystem: "20260101_adhoc.sql" +``` + +#### Step 2: Decide ā€” restore or remove the orphan + +If the migration **must be preserved** (it created schema your app depends on): +```bash +# Recreate the migration file from git history or database introspection +git log --all --oneline -- supabase/migrations/ +# Or inspect the database directly to recreate the SQL +``` + +If the migration is **safe to remove** (temporary, already superseded): +```bash +# Remove from supabase_migrations_history via the Supabase dashboard +# Dashboard > Table Editor > supabase_migrations_history > delete row +``` + +#### Step 3: Verify CI passes +Re-run the `verify-database` job via `ENABLE_DB_VERIFICATION=true` on the PR or push to `main`. + +--- + +## Comprehensive Troubleshooting + +### "Database connection failed" + +```bash +# Verify environment variables +echo $SUPABASE_URL +echo $SUPABASE_SERVICE_ROLE_KEY + +# Test connection +curl -X POST "$SUPABASE_URL/rest/v1/rpc/exec_sql" \ + -H "Authorization: Bearer $SUPABASE_SERVICE_ROLE_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "SELECT 1"}' +``` + +### "Migration syntax error" + +```bash +# Check migration file for SQL syntax errors +cat supabase/migrations/20260529_your_migration.sql + +# Validate SQL +sqlcheck < supabase/migrations/20260529_your_migration.sql + +# Test locally with psql +psql "$DATABASE_URL" < supabase/migrations/20260529_your_migration.sql +``` + +### "Too many migration conflicts to resolve manually" + +```bash +# Rebuild from scratch (caution: production data loss) +# Only for development databases + +# 1. Back up if needed +pg_dump "$DATABASE_URL" > backup.sql + +# 2. Reset and reapply +npm run db:reset +npm run db:migrate + +# Or re-initialize from git +rm -rf backend/migrations supabase/migrations +git checkout backend/migrations supabase/migrations +npm run db:migrate +``` + +--- + +## Prevention Tips + +1. **Use a single source of truth**: Choose either `backend/migrations/` OR `supabase/migrations/`, not both +2. **Automate checks**: Ensure drift check runs in CI on every PR +3. **Review migration history**: Before merging, inspect actual migrations: + ```bash + git diff HEAD~1..HEAD -- backend/migrations/ supabase/migrations/ + ``` +4. **Test locally first**: Always apply migrations locally before opening a PR: + ```bash + npm run db:migrate + npm run check:migrations --verify-db + ``` +5. **Document schema changes**: Add comments to migrations explaining what they do +6. **Monitor CI**: Set up alerts for failed migration checks + +--- + +## Getting Help + +If you encounter issues not covered here: + +1. Check recent migration files for obvious errors +2. Review git log for recent changes: + ```bash + git log -p -- backend/migrations/ supabase/migrations/ | head -100 + ``` +3. Run the validation script in verbose mode: + ```bash + node scripts/validate-migration-state.js --json | jq '.issues[] | select(.severity=="error")' + ``` +4. Consult the database logs: + ```bash + supabase logs + ``` + +--- + +## Related Commands + +```bash +# File-only drift check (fastest) +npm run check:migrations + +# File + database drift check +npm run check:migrations --verify-db + +# Validate current database state +node scripts/validate-migration-state.js + +# Create a new migration +npm run db:new + +# Apply migrations +npm run db:migrate + +# Reset local database (development only) +npm run db:reset +``` diff --git a/scripts/check-migration-drift.js b/scripts/check-migration-drift.js index 21c90658..23e07e73 100644 --- a/scripts/check-migration-drift.js +++ b/scripts/check-migration-drift.js @@ -3,10 +3,21 @@ /** * Migration Drift Check Script * - * Detects drift between backend/migrations and supabase/migrations folders. - * This script ensures schema changes cannot silently diverge between folders. + * Detects drift between backend/migrations and supabase/migrations folders, + * and optionally verifies applied migrations against the database. * - * Usage: node scripts/check-migration-drift.js + * Usage: + * node scripts/check-migration-drift.js # File-only check + * node scripts/check-migration-drift.js --verify-db # File + database verification + * node scripts/check-migration-drift.js --json # JSON output format + * node scripts/check-migration-drift.js --verify-db --json # DB verification with JSON output + * node scripts/check-migration-drift.js --verify-db --env local # Label output as local environment + * node scripts/check-migration-drift.js --verify-db --env ci-remote # Label output as CI remote environment + * + * Environment variables (for --verify-db): + * SUPABASE_URL: Supabase project URL + * SUPABASE_SERVICE_ROLE_KEY: Service role key for database access + * DATABASE_URL: Optional direct Postgres connection string (overrides Supabase) * * Exit codes: * 0 - No drift detected @@ -16,11 +27,96 @@ const fs = require('fs'); const path = require('path'); +const https = require('https'); +const http = require('http'); // Configuration const BACKEND_MIGRATIONS = path.join(__dirname, '..', 'backend', 'migrations'); const SUPABASE_MIGRATIONS = path.join(__dirname, '..', 'supabase', 'migrations'); +// Parse command-line arguments +const args = process.argv.slice(2); +const envIdx = args.indexOf('--env'); +const options = { + verifyDb: args.includes('--verify-db'), + json: args.includes('--json'), + env: envIdx >= 0 && args[envIdx + 1] ? args[envIdx + 1] : 'unknown' +}; + +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ +// Database Query Utilities +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Execute a SQL query against Supabase via REST API + * @param {string} query - SQL query to execute + * @param {string} supabaseUrl - Supabase project URL + * @param {string} supabaseKey - Service role key + * @returns {Promise} Query result + */ +async function executeSupabaseQuery(query, supabaseUrl, supabaseKey) { + return new Promise((resolve, reject) => { + const body = JSON.stringify({ query }); + const url = new URL(`${supabaseUrl}/rest/v1/rpc/exec_sql`); + + const options = { + hostname: url.hostname, + port: url.port || (url.protocol === 'https:' ? 443 : 80), + path: url.pathname + url.search, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${supabaseKey}`, + 'Content-Length': Buffer.byteLength(body), + }, + }; + + const lib = url.protocol === 'https:' ? https : http; + const req = lib.request(options, (res) => { + let data = ''; + res.on('data', (chunk) => { data += chunk; }); + res.on('end', () => { + try { + const parsed = JSON.parse(data); + if (res.statusCode >= 400) { + reject(new Error(`Query failed: ${parsed.message || data}`)); + } else { + resolve(parsed); + } + } catch { + reject(new Error(`Failed to parse response: ${data}`)); + } + }); + }); + + req.on('error', reject); + req.write(body); + req.end(); + }); +} + +/** + * Fetch applied migrations from database + * @param {string} supabaseUrl - Supabase project URL + * @param {string} supabaseKey - Service role key + * @returns {Promise} List of applied migrations + */ +async function getAppliedMigrations(supabaseUrl, supabaseKey) { + try { + const result = await executeSupabaseQuery( + `SELECT name, executed_at FROM supabase_migrations_history ORDER BY name ASC`, + supabaseUrl, + supabaseKey + ); + return (result || []).map(row => ({ + name: row.name || row.Name || row[0], + executedAt: row.executed_at || row.ExecutedAt || row[1] + })); + } catch (err) { + throw new Error(`Failed to fetch migration history: ${err.message}`); + } +} + // Normalize SQL content for comparison (remove comments, whitespace, case) function normalizeSQL(content) { return content @@ -131,7 +227,9 @@ function compareMigrations(name1, m1, name2, m2) { // Main drift detection function function detectDrift() { - console.log('šŸ” Checking migration drift between backend and supabase...\n'); + if (!options.json) { + console.log(`šŸ” Checking migration drift between backend and supabase... [env: ${options.env}]\n`); + } const backendMigrations = readMigrations(BACKEND_MIGRATIONS); const supabaseMigrations = readMigrations(SUPABASE_MIGRATIONS); @@ -182,62 +280,210 @@ function detectDrift() { } // Report findings - console.log('=== Migration Analysis ===\n'); - console.log(`Backend migrations: ${backendMigrations.size} files`); - console.log(`Supabase migrations: ${supabaseMigrations.size} files\n`); + if (!options.json) { + console.log('=== Migration Analysis ===\n'); + console.log(`Backend migrations: ${backendMigrations.size} files`); + console.log(`Supabase migrations: ${supabaseMigrations.size} files\n`); + } if (issues.length === 0) { - console.log('āœ… No migration drift detected.'); - console.log('\n--- Summary ---'); - console.log(`Total backend tables: ${allBackendTables.size}`); - console.log(`Total supabase tables: ${allSupabaseTables.size}`); - return { success: true, issues: [] }; + if (!options.json) { + console.log('āœ… No migration drift detected.'); + console.log('\n--- Summary ---'); + console.log(`Total backend tables: ${allBackendTables.size}`); + console.log(`Total supabase tables: ${allSupabaseTables.size}`); + } + return { + success: true, + issues: [], + fileCheck: { + backendCount: backendMigrations.size, + supabaseCount: supabaseMigrations.size, + backendTableCount: allBackendTables.size, + supabaseTableCount: allSupabaseTables.size + } + }; } // Group issues by type const errors = issues.filter(i => i.severity === 'error'); const warnings = issues.filter(i => i.severity === 'warning'); - if (errors.length > 0) { - console.log('āŒ ERRORS (must fix):\n'); - for (const issue of errors) { - console.log(` [${issue.type.toUpperCase()}] ${issue.message}`); + if (!options.json) { + if (errors.length > 0) { + console.log('āŒ ERRORS (must fix):\n'); + for (const issue of errors) { + console.log(` [${issue.type.toUpperCase()}] ${issue.message}`); + } + console.log(''); } - console.log(''); - } - - if (warnings.length > 0) { - console.log('āš ļø WARNINGS (review recommended):\n'); - for (const issue of warnings) { - console.log(` [${issue.type.toUpperCase()}] ${issue.message}`); + + if (warnings.length > 0) { + console.log('āš ļø WARNINGS (review recommended):\n'); + for (const issue of warnings) { + console.log(` [${issue.type.toUpperCase()}] ${issue.message}`); + } + console.log(''); } - console.log(''); + + console.log('\n--- Recommendations ---'); + console.log('1. Review duplicate migrations and consolidate them'); + console.log('2. Ensure all schema changes go through a single migration path'); + console.log('3. Use either backend/migrations OR supabase/migrations, not both'); + console.log('4. Run this check in CI to prevent drift'); } - console.log('\n--- Recommendations ---'); - console.log('1. Review duplicate migrations and consolidate them'); - console.log('2. Ensure all schema changes go through a single migration path'); - console.log('3. Use either backend/migrations OR supabase/migrations, not both'); - console.log('4. Run this check in CI to prevent drift'); - return { success: errors.length === 0, issues, - summary: { + fileCheck: { + backendCount: backendMigrations.size, + supabaseCount: supabaseMigrations.size, + backendTableCount: allBackendTables.size, + supabaseTableCount: allSupabaseTables.size, errors: errors.length, - warnings: warnings.length, - backendMigrations: backendMigrations.size, - supabaseMigrations: supabaseMigrations.size + warnings: warnings.length } }; } +/** + * Verify database state against filesystem migrations + * @returns {Promise} Database verification result + */ +async function verifyDatabaseState() { + const supabaseUrl = process.env.SUPABASE_URL; + const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY; + + if (!supabaseUrl || !supabaseKey) { + const msg = 'SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY environment variables required for --verify-db'; + if (options.json) { + return { + success: false, + error: msg, + dbCheck: null + }; + } + throw new Error(msg); + } + + try { + const appliedMigrations = await getAppliedMigrations(supabaseUrl, supabaseKey); + const appliedNames = new Set(appliedMigrations.map(m => m.name)); + + // Get all filesystem migrations + const backendMigrations = readMigrations(BACKEND_MIGRATIONS); + const supabaseMigrations = readMigrations(SUPABASE_MIGRATIONS); + const allFileSystemMigrations = new Set([ + ...backendMigrations.keys(), + ...supabaseMigrations.keys() + ]); + + const dbIssues = []; + + // Check for migrations in database but not in filesystem + for (const appliedMigration of appliedMigrations) { + if (!allFileSystemMigrations.has(appliedMigration.name)) { + dbIssues.push({ + type: 'orphaned_migration', + severity: 'warning', + message: `Migration in database but not in filesystem: "${appliedMigration.name}"`, + migration: appliedMigration.name, + executedAt: appliedMigration.executedAt + }); + } + } + + // Check for migrations in filesystem but not in database + for (const fileMigration of allFileSystemMigrations) { + if (!appliedNames.has(fileMigration)) { + dbIssues.push({ + type: 'unapplied_migration', + severity: 'warning', + message: `Migration in filesystem but not applied to database: "${fileMigration}"`, + migration: fileMigration + }); + } + } + + if (!options.json) { + if (dbIssues.length === 0) { + console.log('āœ… Database migration state matches filesystem.'); + console.log(`Applied migrations: ${appliedMigrations.length}`); + } else { + console.log('\n=== Database Verification Issues ===\n'); + for (const issue of dbIssues) { + console.log(` [${issue.type.toUpperCase()}] ${issue.message}`); + } + } + } + + return { + success: dbIssues.filter(i => i.severity === 'error').length === 0, + dbCheck: { + appliedCount: appliedMigrations.length, + filesystemCount: allFileSystemMigrations.size, + issues: dbIssues, + appliedMigrations: appliedMigrations.map(m => ({ name: m.name, executedAt: m.executedAt })) + } + }; + } catch (err) { + const msg = `Database verification failed: ${err.message}`; + if (!options.json) { + console.error(`āš ļø ${msg}`); + console.error('Proceeding with file-level checks only.'); + } + return { + success: true, + dbCheck: { error: msg, appliedCount: 0, filesystemCount: 0 } + }; + } +} + // Run the check -const result = detectDrift(); - -if (!result.success) { - console.log('\nāŒ Migration drift detected! Please fix the issues above.'); - process.exit(1); -} else { - process.exit(0); -} \ No newline at end of file +async function main() { + try { + const fileCheckResult = detectDrift(); + + let dbCheckResult = null; + if (options.verifyDb) { + dbCheckResult = await verifyDatabaseState(); + } + + // Determine overall success + const fileSuccess = fileCheckResult.success; + const dbSuccess = dbCheckResult ? dbCheckResult.success : true; + const overallSuccess = fileSuccess && dbSuccess; + + // Output JSON if requested + if (options.json) { + const output = { + success: overallSuccess, + env: options.env, + fileCheck: fileCheckResult.fileCheck, + dbCheck: dbCheckResult?.dbCheck || null, + issues: fileCheckResult.issues + }; + console.log(JSON.stringify(output, null, 2)); + } else if (!overallSuccess && !options.json) { + console.log('\nāŒ Migration drift detected! Please fix the issues above.'); + if (options.verifyDb) { + console.log('\nFor remediation guidance, see docs/MIGRATION_REMEDIATION.md'); + } + } + + process.exit(overallSuccess ? 0 : 1); + } catch (err) { + if (options.json) { + console.log(JSON.stringify({ + success: false, + error: err.message + }, null, 2)); + } else { + console.error(`\nāŒ Error: ${err.message}`); + } + process.exit(2); + } +} + +main(); \ No newline at end of file diff --git a/scripts/validate-migration-state.js b/scripts/validate-migration-state.js new file mode 100644 index 00000000..70dddefd --- /dev/null +++ b/scripts/validate-migration-state.js @@ -0,0 +1,287 @@ +#!/usr/bin/env node + +/** + * Migration State Validator + * + * Queries the database and reports applied migration state. + * Useful for debugging, comparing across environments, and validating migration history. + * + * Usage: + * node scripts/validate-migration-state.js # Against SUPABASE_URL + * node scripts/validate-migration-state.js --json # JSON output + * node scripts/validate-migration-state.js --compare-files # Compare with filesystem + * node scripts/validate-migration-state.js --env production # Use production DB URL + * + * Environment variables: + * SUPABASE_URL: Supabase project URL (default) + * SUPABASE_SERVICE_ROLE_KEY: Service role key + * PRODUCTION_DB_URL: Direct Postgres URL for production validation + * + * Exit codes: + * 0 - Validation successful + * 1 - Validation issues found + * 2 - Error occurred + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const https = require('https'); +const http = require('http'); + +// Parse arguments +const args = process.argv.slice(2); +const options = { + json: args.includes('--json'), + compareFiles: args.includes('--compare-files'), + env: args.includes('--env') ? args[args.indexOf('--env') + 1] : 'default' +}; + +const BACKEND_MIGRATIONS = path.join(__dirname, '..', 'backend', 'migrations'); +const SUPABASE_MIGRATIONS = path.join(__dirname, '..', 'supabase', 'migrations'); + +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ +// Database Query Utilities +// ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ + +/** + * Execute SQL query via Supabase REST API + */ +async function executeQuery(query, supabaseUrl, supabaseKey) { + return new Promise((resolve, reject) => { + const body = JSON.stringify({ query }); + const url = new URL(`${supabaseUrl}/rest/v1/rpc/exec_sql`); + + const options = { + hostname: url.hostname, + port: url.port || (url.protocol === 'https:' ? 443 : 80), + path: url.pathname + url.search, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${supabaseKey}`, + 'Content-Length': Buffer.byteLength(body), + }, + }; + + const lib = url.protocol === 'https:' ? https : http; + const req = lib.request(options, (res) => { + let data = ''; + res.on('data', (chunk) => { data += chunk; }); + res.on('end', () => { + try { + const parsed = JSON.parse(data); + if (res.statusCode >= 400) { + reject(new Error(`Query failed: ${parsed.message || data}`)); + } else { + resolve(parsed); + } + } catch { + reject(new Error(`Failed to parse response: ${data}`)); + } + }); + }); + + req.on('error', reject); + req.write(body); + req.end(); + }); +} + +/** + * Get migration history from database + */ +async function getMigrationHistory(supabaseUrl, supabaseKey) { + try { + const result = await executeQuery( + `SELECT + name, + executed_at, + execution_time_ms, + success + FROM supabase_migrations_history + ORDER BY name ASC`, + supabaseUrl, + supabaseKey + ); + + return (result || []).map(row => ({ + name: row.name || row.Name || row[0], + executedAt: row.executed_at || row.ExecutedAt || row[1], + executionTimeMs: row.execution_time_ms || row.ExecutionTimeMs || row[2], + success: row.success !== false && (row.Success !== false) && row[3] !== false + })); + } catch (err) { + throw new Error(`Failed to fetch migration history: ${err.message}`); + } +} + +/** + * Get database schema tables + */ +async function getDatabaseTables(supabaseUrl, supabaseKey) { + try { + const result = await executeQuery( + `SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + ORDER BY table_name ASC`, + supabaseUrl, + supabaseKey + ); + + return (result || []).map(row => row.table_name || row.TableName || row[0]); + } catch (err) { + throw new Error(`Failed to fetch database tables: ${err.message}`); + } +} + +/** + * Read migrations from filesystem + */ +function readFilesystemMigrations() { + const migrations = { + backend: [], + supabase: [] + }; + + // Read backend migrations + if (fs.existsSync(BACKEND_MIGRATIONS)) { + migrations.backend = fs.readdirSync(BACKEND_MIGRATIONS) + .filter(f => f.endsWith('.sql')) + .sort(); + } + + // Read supabase migrations + if (fs.existsSync(SUPABASE_MIGRATIONS)) { + migrations.supabase = fs.readdirSync(SUPABASE_MIGRATIONS) + .filter(f => f.endsWith('.sql')) + .sort(); + } + + return migrations; +} + +/** + * Validate migration state + */ +async function validateMigrationState() { + const supabaseUrl = process.env.SUPABASE_URL; + const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY; + + if (!supabaseUrl || !supabaseKey) { + throw new Error('SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY environment variables required'); + } + + if (!options.json) { + console.log('šŸ“Š Migration State Validator\n'); + console.log(`Environment: ${options.env}`); + console.log(`Database: ${supabaseUrl}\n`); + } + + // Fetch database state + const appliedMigrations = await getMigrationHistory(supabaseUrl, supabaseKey); + const tables = await getDatabaseTables(supabaseUrl, supabaseKey); + + const appliedNames = new Set(appliedMigrations.map(m => m.name)); + const issues = []; + + if (!options.json) { + console.log('=== Database State ===\n'); + console.log(`Applied migrations: ${appliedMigrations.length}`); + console.log(`Database tables: ${tables.length}`); + console.log('\n--- Applied Migrations ---'); + for (const migration of appliedMigrations) { + const status = migration.success ? 'āœ…' : 'āŒ'; + console.log(`${status} ${migration.name}`); + if (migration.executedAt) { + console.log(` Executed: ${migration.executedAt}`); + } + } + } + + // Compare with filesystem if requested + if (options.compareFiles) { + const fsRaigrations = readFilesystemMigrations(); + const allFsFiles = new Set([...fsRaigrations.backend, ...fsRaigrations.supabase]); + + if (!options.json) { + console.log('\n=== Filesystem Comparison ===\n'); + } + + // Check for missing migrations in database + for (const file of allFsFiles) { + if (!appliedNames.has(file)) { + issues.push({ + type: 'unapplied', + severity: 'warning', + migration: file, + message: `Migration in filesystem but not applied to database: "${file}"` + }); + if (!options.json) { + console.log(`āš ļø NOT APPLIED: ${file}`); + } + } + } + + // Check for orphaned migrations in database + for (const migration of appliedMigrations) { + if (!allFsFiles.has(migration.name)) { + issues.push({ + type: 'orphaned', + severity: 'warning', + migration: migration.name, + message: `Migration in database but not in filesystem: "${migration.name}"` + }); + if (!options.json) { + console.log(`āš ļø ORPHANED: ${migration.name}`); + } + } + } + + if (issues.length === 0 && !options.json) { + console.log('āœ… Filesystem and database migrations are in sync'); + } + } + + if (!options.json && issues.length === 0) { + console.log('\nāœ… Migration state validation successful'); + } + + return { + success: true, + appliedMigrations, + tables, + issues, + environment: options.env + }; +} + +/** + * Main entry point + */ +async function main() { + try { + const result = await validateMigrationState(); + + if (options.json) { + console.log(JSON.stringify(result, null, 2)); + } + + const hasErrors = result.issues.some(i => i.severity === 'error'); + process.exit(hasErrors ? 1 : 0); + } catch (err) { + if (options.json) { + console.log(JSON.stringify({ + success: false, + error: err.message + }, null, 2)); + } else { + console.error(`\nāŒ Error: ${err.message}`); + } + process.exit(2); + } +} + +main(); diff --git a/scripts/validate-script-syntax.js b/scripts/validate-script-syntax.js new file mode 100644 index 00000000..56b2e1c1 --- /dev/null +++ b/scripts/validate-script-syntax.js @@ -0,0 +1,23 @@ +#!/usr/bin/env node + +/** + * Quick syntax validator for the drift check script + */ + +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const scriptPath = path.join(__dirname, 'check-migration-drift.js'); +const scriptContent = fs.readFileSync(scriptPath, 'utf-8'); + +try { + // Try to compile the script + new vm.Script(scriptContent); + console.log('āœ… check-migration-drift.js: Syntax valid'); + process.exit(0); +} catch (err) { + console.error('āŒ Syntax error in check-migration-drift.js:'); + console.error(err.message); + process.exit(1); +}