diff --git a/.context/sync-and-readiness.md b/.context/sync-and-readiness.md new file mode 100644 index 0000000..0264497 --- /dev/null +++ b/.context/sync-and-readiness.md @@ -0,0 +1,68 @@ +# Sync & Readiness + +## How Sync Works + +1. **Pull from GCP**: `npm run db:pull:dump` exports schema + data from GCP source of truth into `dumps/schema.sql` and `dumps/data.sql` (gitignored). +2. **Sync to targets**: `npm run db:sync:` destroys the target database and re-applies schema + data from the dumps. + - Supported targets: `local`, `neon`, `supabase`, `all` + - `gcp` is always refused (read-only source) +3. **Safety**: Requires either `DB_SYNC_ALLOW_DESTRUCTIVE=true` env var or interactive `y/N` confirmation. +4. **Verification**: After sync completes, the script prints table count and key row counts (users, attendance). + +### Scripts + +```bash +npm run db:sync:local # Sync dumps -> local Docker +npm run db:sync:neon # Sync dumps -> Neon +npm run db:sync:supabase # Sync dumps -> Supabase +npm run db:sync:all # Sync dumps -> all three targets sequentially +``` + +### Prerequisites + +- `dumps/schema.sql` and `dumps/data.sql` must exist (run `npm run db:pull:dump` first) +- Target env vars must be set in `.env.local` (`DATABASE_URL_LOCAL`, `DATABASE_URL_NEON`, `DATABASE_URL_SUPABASE`) + +## Readiness Check API + +### `GET /api/readiness` + +Returns an array of platform readiness statuses: + +```json +[ + { + "platformId": "gcp", + "name": "GCP (Source)", + "tableCount": 43, + "sampleRowCount": 54000, + "ready": true + }, + { + "platformId": "neon", + "name": "Neon", + "tableCount": 0, + "sampleRowCount": 0, + "ready": false + } +] +``` + +- Checks `information_schema.tables` for table count +- Checks `public.users` row count as a sentinel +- `ready = tableCount > 0 && sampleRowCount > 0` +- Each platform check is wrapped in try/catch (unreachable platforms return `ready: false`) + +### UI Integration + +- Platform pills show an amber warning dot when a platform is not ready +- Warning cards appear below the pill selector for selected platforms that are not ready +- The DataDiff component shows diagnostic hints when one side returns 0 rows +- The Performance Chart shows warning banners when row counts differ + +## Troubleshooting + +1. **"dumps not found" error**: Run `npm run db:pull:dump` to export from GCP +2. **Env var not set**: Add `DATABASE_URL_` to `.env.local` +3. **Platform shows 0 tables after sync**: Check that `db-migrate.sh` and `db-seed.sh` completed without errors. Run `npm run db:verify:` for details. +4. **Neon/Supabase connection timeout**: Check that IP allowlists include your current IP diff --git a/CLAUDE.md b/CLAUDE.md index 7f7c9e3..66cb77b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,6 +25,10 @@ npm run db:verify:local # Health check local Docker npm run db:verify:gcp # Health check GCP npm run db:verify:neon # Health check Neon npm run db:verify:supabase # Health check Supabase +npm run db:sync:local # Reset + migrate + seed local from dumps +npm run db:sync:neon # Reset + migrate + seed Neon from dumps +npm run db:sync:supabase # Reset + migrate + seed Supabase from dumps +npm run db:sync:all # Sync all targets sequentially npm run db:generate # Drizzle generate migrations npm run db:migrate:metadata # Push schema to metadata DB npm run db:studio # Drizzle Studio GUI @@ -55,6 +59,7 @@ npm run db:studio # Drizzle Studio GUI - `POST /api/compare/schema` — {leftId, rightId} -> schema diff - `POST /api/cron/latency` — Collect health snapshots (QStash target) - `GET /api/analytics/latency` — Query latency history + stats +- `GET /api/readiness` — Platform readiness (table count, sample rows, ready flag) ## Env Variables @@ -80,3 +85,4 @@ npm run db:studio # Drizzle Studio GUI - `architecture.md` — Key patterns, env vars, API routes - `status.md` — Phase progress tracker - `pax-vault-patterns.md` — Patterns borrowed from PAX-VAULT +- `sync-and-readiness.md` — Sync scripts + readiness API docs diff --git a/package.json b/package.json index 0348f0d..f6a5f79 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,10 @@ "db:verify:neon": "tsx scripts/db-verify.ts neon", "db:verify:supabase": "tsx scripts/db-verify.ts supabase", "db:verify:gcp": "tsx scripts/db-verify.ts gcp", + "db:sync:local": "tsx scripts/db-sync.ts local", + "db:sync:neon": "tsx scripts/db-sync.ts neon", + "db:sync:supabase": "tsx scripts/db-sync.ts supabase", + "db:sync:all": "tsx scripts/db-sync.ts all", "firebase:env": "bash scripts/firebase-env.sh", "db:generate": "drizzle-kit generate", "db:migrate:metadata": "drizzle-kit push", diff --git a/scripts/db-sync.ts b/scripts/db-sync.ts new file mode 100644 index 0000000..1fee8c6 --- /dev/null +++ b/scripts/db-sync.ts @@ -0,0 +1,224 @@ +import { Pool } from "pg"; +import { resolve } from "path"; +import { config } from "dotenv"; +import { execSync } from "child_process"; +import { createInterface } from "readline"; +import { existsSync } from "fs"; + +// Load .env.local +config({ path: resolve(__dirname, "../.env.local") }); + +const target = process.argv[2]; + +if (!target) { + console.error("Usage: db-sync.ts "); + console.error("Targets: local, neon, supabase, all"); + process.exit(1); +} + +if (target === "gcp") { + console.error( + "ERROR: Refusing to sync to GCP — it is the read-only source of truth.", + ); + console.error("Use `npm run db:pull:dump` to pull data FROM GCP."); + process.exit(1); +} + +const validTargets = ["local", "neon", "supabase", "all"]; +if (!validTargets.includes(target)) { + console.error(`Unknown target: ${target}. Use: ${validTargets.join(", ")}`); + process.exit(1); +} + +const envKeyMap: Record = { + local: "DATABASE_URL_LOCAL", + neon: "DATABASE_URL_NEON", + supabase: "DATABASE_URL_SUPABASE", +}; + +const dumpDir = resolve(__dirname, "../dumps"); +const schemaPath = resolve(dumpDir, "schema.sql"); +const dataPath = resolve(dumpDir, "data.sql"); + +function confirm(message: string): Promise { + const rl = createInterface({ input: process.stdin, output: process.stdout }); + return new Promise((resolve) => { + rl.question(`${message} [y/N] `, (answer) => { + rl.close(); + resolve(answer.trim().toLowerCase() === "y"); + }); + }); +} + +function run(cmd: string) { + console.log(`$ ${cmd}`); + execSync(cmd, { stdio: "inherit", cwd: resolve(__dirname, "..") }); +} + +async function resetLocal() { + console.log("Tearing down Docker volumes and restarting..."); + run("docker compose down -v"); + run("docker compose up -d"); + + console.log("Waiting for Postgres to be ready..."); + const maxAttempts = 30; + for (let i = 0; i < maxAttempts; i++) { + try { + execSync("docker compose exec -T postgres pg_isready -U postgres", { + stdio: "pipe", + cwd: resolve(__dirname, ".."), + }); + console.log("Postgres is ready."); + break; + } catch { + if (i === maxAttempts - 1) { + console.error("Postgres did not become ready in time."); + process.exit(1); + } + await new Promise((r) => setTimeout(r, 1000)); + } + } + + run("bash scripts/db-migrate.sh local"); + run("bash scripts/db-seed.sh local"); +} + +async function resetRemote(t: string) { + const envKey = envKeyMap[t]; + const connectionString = process.env[envKey]; + if (!connectionString) { + console.error(`${envKey} is not set. Add it to .env.local`); + process.exit(1); + } + + const pool = new Pool({ connectionString, connectionTimeoutMillis: 10000 }); + + try { + console.log(`Dropping schemas on ${t}...`); + await pool.query("DROP SCHEMA IF EXISTS public CASCADE"); + await pool.query("DROP SCHEMA IF EXISTS codex CASCADE"); + await pool.query("DROP SCHEMA IF EXISTS regionpages CASCADE"); + await pool.query("CREATE SCHEMA public"); + console.log("Schemas dropped and public recreated."); + } finally { + await pool.end(); + } + + run(`bash scripts/db-migrate.sh ${t}`); + run(`bash scripts/db-seed.sh ${t}`); +} + +async function verify(t: string) { + const envKey = envKeyMap[t]; + const connectionString = process.env[envKey]; + if (!connectionString) { + console.error(`${envKey} is not set — skipping verification.`); + return; + } + + const pool = new Pool({ connectionString, connectionTimeoutMillis: 10000 }); + + try { + const tablesResult = await pool.query(` + SELECT COUNT(*) as count + FROM information_schema.tables + WHERE table_schema NOT IN ('information_schema', 'pg_catalog') + AND table_type = 'BASE TABLE' + `); + const tableCount = tablesResult.rows[0].count; + + let userCount = "N/A"; + try { + const usersResult = await pool.query( + "SELECT COUNT(*) as count FROM public.users", + ); + userCount = usersResult.rows[0].count; + } catch { + // users table may not exist + } + + let attendanceCount = "N/A"; + try { + const attendanceResult = await pool.query( + "SELECT COUNT(*) as count FROM public.attendance", + ); + attendanceCount = attendanceResult.rows[0].count; + } catch { + // attendance table may not exist + } + + console.log(`\n Verification for ${t}:`); + console.log(` Tables: ${tableCount}`); + console.log(` Users: ${userCount}`); + console.log(` Attendance: ${attendanceCount}`); + } catch (err) { + console.error( + ` Verification failed for ${t}:`, + err instanceof Error ? err.message : err, + ); + } finally { + await pool.end(); + } +} + +async function syncTarget(t: string) { + const start = performance.now(); + console.log(`\n========== Syncing ${t} ==========\n`); + + if (t === "local") { + await resetLocal(); + } else { + await resetRemote(t); + } + + await verify(t); + + const duration = ((performance.now() - start) / 1000).toFixed(1); + console.log(`\n ${t} sync complete in ${duration}s`); +} + +async function main() { + // Check dumps exist + if (!existsSync(schemaPath)) { + console.error(`ERROR: ${schemaPath} not found.`); + console.error("Run `npm run db:pull:dump` first to pull from GCP."); + process.exit(1); + } + if (!existsSync(dataPath)) { + console.error(`ERROR: ${dataPath} not found.`); + console.error("Run `npm run db:pull:dump` first to pull from GCP."); + process.exit(1); + } + + const targets = target === "all" ? ["local", "neon", "supabase"] : [target]; + + // Verify env vars for all targets + for (const t of targets) { + const envKey = envKeyMap[t]; + if (!process.env[envKey]) { + console.error(`${envKey} is not set. Add it to .env.local`); + process.exit(1); + } + } + + // Safety check + const allowDestructive = process.env.DB_SYNC_ALLOW_DESTRUCTIVE === "true"; + + if (!allowDestructive) { + const confirmed = await confirm( + `Sync ${targets.join(", ")}? This will DESTROY all existing data on these targets.`, + ); + if (!confirmed) { + console.log("Aborted."); + process.exit(0); + } + } + + for (const t of targets) { + await syncTarget(t); + } + + console.log("\nAll syncs complete."); +} + +main(); diff --git a/src/__tests__/schema-compare.test.ts b/src/__tests__/schema-compare.test.ts new file mode 100644 index 0000000..657853a --- /dev/null +++ b/src/__tests__/schema-compare.test.ts @@ -0,0 +1,236 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +// Mock the platforms module (imported by the route for side effects) +vi.mock("@/lib/platforms", () => ({})); + +// Mock the registry +const mockGetPlatform = vi.fn(); +vi.mock("@/lib/platforms/registry", () => ({ + getPlatform: (...args: unknown[]) => mockGetPlatform(...args), +})); + +function makePlatform(overrides: { + id: string; + name: string; + configured?: boolean; + schema?: { + tables: Array<{ + name: string; + schema: string; + columns: Array<{ + name: string; + dataType: string; + isNullable: boolean; + columnDefault: string | null; + ordinalPosition: number; + }>; + }>; + latencyMs: number; + }; + schemaError?: Error; +}) { + return { + id: overrides.id, + name: overrides.name, + isConfigured: () => overrides.configured ?? true, + getSchema: overrides.schemaError + ? vi.fn().mockRejectedValue(overrides.schemaError) + : vi + .fn() + .mockResolvedValue(overrides.schema ?? { tables: [], latencyMs: 5 }), + }; +} + +function makeRequest(body: Record) { + return new Request("http://localhost:3002/api/compare/schema", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); +} + +describe("Schema Compare API", () => { + beforeEach(() => { + mockGetPlatform.mockReset(); + }); + + it("returns 400 when leftId or rightId is missing", async () => { + const { POST } = await import("@/app/api/compare/schema/route"); + const res = await POST(makeRequest({ leftId: "", rightId: "neon" })); + expect(res.status).toBe(400); + const json = await res.json(); + expect(json.error).toContain("required"); + }); + + it("returns 404 for unknown platform", async () => { + mockGetPlatform.mockReturnValue(undefined); + const { POST } = await import("@/app/api/compare/schema/route"); + const res = await POST(makeRequest({ leftId: "unknown", rightId: "neon" })); + expect(res.status).toBe(404); + }); + + it("returns 500 when getSchema throws", async () => { + const left = makePlatform({ + id: "gcp", + name: "GCP", + schemaError: new Error("connection refused"), + }); + const right = makePlatform({ id: "neon", name: "Neon" }); + + mockGetPlatform.mockImplementation((id: string) => { + if (id === "gcp") return left; + if (id === "neon") return right; + return undefined; + }); + + const { POST } = await import("@/app/api/compare/schema/route"); + const res = await POST(makeRequest({ leftId: "gcp", rightId: "neon" })); + expect(res.status).toBe(500); + const json = await res.json(); + expect(json.error).toContain("connection refused"); + }); + + it("builds correct diff for matching schemas", async () => { + const schema = { + tables: [ + { + name: "users", + schema: "public", + columns: [ + { + name: "id", + dataType: "integer", + isNullable: false, + columnDefault: null, + ordinalPosition: 1, + }, + { + name: "name", + dataType: "text", + isNullable: true, + columnDefault: null, + ordinalPosition: 2, + }, + ], + }, + ], + latencyMs: 10, + }; + + const left = makePlatform({ id: "gcp", name: "GCP", schema }); + const right = makePlatform({ id: "neon", name: "Neon", schema }); + + mockGetPlatform.mockImplementation((id: string) => { + if (id === "gcp") return left; + if (id === "neon") return right; + return undefined; + }); + + const { POST } = await import("@/app/api/compare/schema/route"); + const res = await POST(makeRequest({ leftId: "gcp", rightId: "neon" })); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.diff).toHaveLength(1); + expect(json.diff[0].status).toBe("match"); + expect(json.diff[0].table).toBe("public.users"); + }); + + it("detects left_only and right_only tables", async () => { + const leftSchema = { + tables: [{ name: "users", schema: "public", columns: [] }], + latencyMs: 5, + }; + const rightSchema = { + tables: [{ name: "posts", schema: "public", columns: [] }], + latencyMs: 5, + }; + + const left = makePlatform({ id: "gcp", name: "GCP", schema: leftSchema }); + const right = makePlatform({ + id: "neon", + name: "Neon", + schema: rightSchema, + }); + + mockGetPlatform.mockImplementation((id: string) => { + if (id === "gcp") return left; + if (id === "neon") return right; + return undefined; + }); + + const { POST } = await import("@/app/api/compare/schema/route"); + const res = await POST(makeRequest({ leftId: "gcp", rightId: "neon" })); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.diff).toHaveLength(2); + + const leftOnly = json.diff.find( + (d: { table: string }) => d.table === "public.users", + ); + expect(leftOnly.status).toBe("left_only"); + + const rightOnly = json.diff.find( + (d: { table: string }) => d.table === "public.posts", + ); + expect(rightOnly.status).toBe("right_only"); + }); + + it("detects column type differences", async () => { + const leftSchema = { + tables: [ + { + name: "users", + schema: "public", + columns: [ + { + name: "id", + dataType: "integer", + isNullable: false, + columnDefault: null, + ordinalPosition: 1, + }, + ], + }, + ], + latencyMs: 5, + }; + const rightSchema = { + tables: [ + { + name: "users", + schema: "public", + columns: [ + { + name: "id", + dataType: "bigint", + isNullable: false, + columnDefault: null, + ordinalPosition: 1, + }, + ], + }, + ], + latencyMs: 5, + }; + + const left = makePlatform({ id: "gcp", name: "GCP", schema: leftSchema }); + const right = makePlatform({ + id: "neon", + name: "Neon", + schema: rightSchema, + }); + + mockGetPlatform.mockImplementation((id: string) => { + if (id === "gcp") return left; + if (id === "neon") return right; + return undefined; + }); + + const { POST } = await import("@/app/api/compare/schema/route"); + const res = await POST(makeRequest({ leftId: "gcp", rightId: "neon" })); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.diff[0].status).toBe("diff"); + expect(json.diff[0].columns[0].status).toBe("diff"); + }); +}); diff --git a/src/app/api/compare/schema/route.ts b/src/app/api/compare/schema/route.ts index d8b0b3b..4283634 100644 --- a/src/app/api/compare/schema/route.ts +++ b/src/app/api/compare/schema/route.ts @@ -43,10 +43,18 @@ export async function POST(request: Request) { ); } - const [leftSchema, rightSchema] = await Promise.all([ - left.getSchema(), - right.getSchema(), - ]); + let leftSchema, rightSchema; + try { + [leftSchema, rightSchema] = await Promise.all([ + left.getSchema(), + right.getSchema(), + ]); + } catch (err) { + return NextResponse.json( + { error: err instanceof Error ? err.message : "Schema fetch failed" }, + { status: 500 }, + ); + } // Build diff const leftTableMap = new Map( diff --git a/src/app/api/readiness/route.ts b/src/app/api/readiness/route.ts new file mode 100644 index 0000000..9e37e5e --- /dev/null +++ b/src/app/api/readiness/route.ts @@ -0,0 +1,49 @@ +import { NextResponse } from "next/server"; +import "@/lib/platforms"; +import { getConfiguredPlatforms } from "@/lib/platforms/registry"; + +export async function GET() { + const platforms = getConfiguredPlatforms(); + + const results = await Promise.all( + platforms.map(async (p) => { + try { + const tableResult = await p.runQuery(` + SELECT COUNT(*) as count + FROM information_schema.tables + WHERE table_schema NOT IN ('information_schema', 'pg_catalog') + AND table_type = 'BASE TABLE' + `); + const tableCount = Number(tableResult.rows[0]?.count ?? 0); + + let sampleRowCount = 0; + try { + const usersResult = await p.runQuery( + "SELECT COUNT(*) as count FROM public.users", + ); + sampleRowCount = Number(usersResult.rows[0]?.count ?? 0); + } catch { + // users table may not exist + } + + return { + platformId: p.id, + name: p.name, + tableCount, + sampleRowCount, + ready: tableCount > 0 && sampleRowCount > 0, + }; + } catch { + return { + platformId: p.id, + name: p.name, + tableCount: 0, + sampleRowCount: 0, + ready: false, + }; + } + }), + ); + + return NextResponse.json(results); +} diff --git a/src/components/comparison-panel.tsx b/src/components/comparison-panel.tsx index 55191cb..5e34db0 100644 --- a/src/components/comparison-panel.tsx +++ b/src/components/comparison-panel.tsx @@ -4,12 +4,10 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { SchemaDiff } from "./schema-diff"; import { DataDiff } from "./data-diff"; import { PerformanceChart } from "./performance-chart"; -import { LatencyAnalytics } from "./latency-analytics"; import { ComparisonResult, SchemaComparisonResult, } from "@/hooks/use-comparison"; -import type { LatencyAnalyticsState } from "@/hooks/use-latency-analytics"; import { Database, Layers, BarChart3 } from "lucide-react"; interface ComparisonPanelProps { @@ -17,7 +15,8 @@ interface ComparisonPanelProps { dataResult: ComparisonResult | null; performanceResults: ComparisonResult[]; queryNames: string[]; - analyticsState: LatencyAnalyticsState; + activeTab: string; + onTabChange: (tab: string) => void; } export function ComparisonPanel({ @@ -25,15 +24,15 @@ export function ComparisonPanel({ dataResult, performanceResults, queryNames, - analyticsState, + activeTab, + onTabChange, }: ComparisonPanelProps) { return ( - + Data Compare Schema Compare Performance - Analytics @@ -74,10 +73,6 @@ export function ComparisonPanel({ /> )} - - - - ); } diff --git a/src/components/dashboard.tsx b/src/components/dashboard.tsx index 6fb4d9c..ced74b4 100644 --- a/src/components/dashboard.tsx +++ b/src/components/dashboard.tsx @@ -4,6 +4,7 @@ import { useState, useCallback, useEffect, useMemo } from "react"; import { usePlatforms, PlatformStatus } from "@/hooks/use-platforms"; import { useComparison, ComparisonResult } from "@/hooks/use-comparison"; import { useLatencyAnalytics } from "@/hooks/use-latency-analytics"; +import { useReadiness } from "@/hooks/use-readiness"; import { QueryRunner } from "./query-runner"; import { ComparisonPanel } from "./comparison-panel"; import { CompactConnectionStatus } from "./compact-connection-status"; @@ -11,7 +12,7 @@ import { LatencyChart } from "./latency-chart"; import { LatencyStatsRow } from "./latency-stats-row"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { Activity, Loader2, RefreshCw } from "lucide-react"; +import { Activity, AlertTriangle, Loader2, RefreshCw } from "lucide-react"; import { PRESET_QUERIES } from "@/lib/queries"; import { TIME_RANGES, Pill, buildChartData } from "@/lib/latency-constants"; @@ -93,10 +94,12 @@ export function Dashboard() { } = usePlatforms(); const comparison = useComparison(); const analyticsState = useLatencyAnalytics(); + const { readiness, refresh: refreshReadiness } = useReadiness(); const [refreshKey, setRefreshKey] = useState(0); const [leftId, setLeftId] = useState(""); const [rightId, setRightId] = useState(""); + const [activeTab, setActiveTab] = useState("data"); const [performanceResults, setPerformanceResults] = useState< ComparisonResult[] >([]); @@ -106,7 +109,8 @@ export function Dashboard() { setRefreshKey((k) => k + 1); refresh(); analyticsState.refresh(); - }, [refresh, analyticsState]); + refreshReadiness(); + }, [refresh, analyticsState, refreshReadiness]); // Auto-select first two configured platforms (render-time, matching original pattern) const handlePlatformsReady = useCallback(() => { @@ -144,12 +148,14 @@ export function Dashboard() { ...prev, matchingPreset?.name || "Custom Query", ]); + setActiveTab("data"); } }; const handleCompareSchema = async () => { if (!leftId || !rightId) return; await comparison.compareSchema(leftId, rightId); + setActiveTab("schema"); }; // Pill selector: tap to toggle selection @@ -293,13 +299,18 @@ export function Dashboard() { const isLeft = leftId === p.id; const isRight = rightId === p.id; const isSelected = isLeft || isRight; + const platformReady = readiness.find( + (r) => r.platformId === p.id, + ); + const notReady = + platformReady && !platformReady.ready && p.configured; return (