feat: 12-step production migration — DB persistence, APP_MODE gating, full test suite & CI

.github/workflows/ci.yml

@@ -0,0 +1,105 @@
+name: CI — Test & Compliance Gate
+
+on:
+  push:
+    branches: [main, production-infrastructure, staging]
+  pull_request:
+    branches: [main, production-infrastructure]
+
+jobs:
+  # ── Python compliance & unit tests ─────────────────────────────────────────
+  python-tests:
+    name: Python Tests + Governance Sweep
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: pip
+
+      - name: Install dependencies
+        run: |
+          pip install -r compliance_agents/requirements.txt
+          pip install pytest pytest-asyncio httpx
+
+      - name: Run unit tests
+        run: pytest tests/unit/ -v --tb=short
+
+      - name: Run security tests
+        run: pytest tests/security/ -v --tb=short
+
+      - name: Verify audit chain integrity
+        run: |
+          python3 -c "
+          import sys; sys.path.insert(0, '.')
+          from compliance_agents.audit_trail.agent import AuditTrailAgent
+          r = AuditTrailAgent().verify_chain_integrity()
+          print(f'Chain OK: {r[\"integrity_ok\"]} | Verified: {r[\"verified_entries\"]} | Quarantined: {r[\"quarantined_entries\"]}')
+          assert r['integrity_ok'], 'AUDIT CHAIN BROKEN — blocking merge'
+          print('✅ Audit chain integrity verified')
+          "
+
+      - name: Run governance sweep (must pass ≥14/16)
+        run: |
+          python3 -c "
+          import sys; sys.path.insert(0, '.')
+          from compliance_agents.governance.engine import GovernanceEngine
+          result = GovernanceEngine().run_full_sweep(period_hours=24)
+          total = sum(len(fw.controls) for fw in result.frameworks)
+          satisfied = sum(1 for fw in result.frameworks for c in fw.controls if c['status'] == 'satisfied')
+          print(f'Controls satisfied: {satisfied}/{total}')
+          assert satisfied >= 14, f'GOVERNANCE GATE FAILED: only {satisfied}/16 controls satisfied'
+          print('✅ Governance sweep passed')
+          "
+
+  # ── Next.js build + TypeScript check ────────────────────────────────────────
+  nextjs-build:
+    name: Next.js Build
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: apps/gem-atr-digital-easyway
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: npm
+          cache-dependency-path: apps/gem-atr-digital-easyway/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: TypeScript check
+        run: npx tsc --noEmit
+
+      - name: Build
+        run: npm run build
+        env:
+          APP_MODE: mock
+          NEXT_PUBLIC_MOCK_MODE: 'false'
+          DATABASE_URL: ./gem-atr.db
+
+  # ── Docker build validation ──────────────────────────────────────────────────
+  docker-build:
+    name: Docker Image Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build compliance_agents image
+        run: docker build -t gem-compliance-agents:ci ./compliance_agents
+
+      - name: Build card_platform_service image
+        run: docker build -t gem-card-platform:ci ./card_platform_service
+
+      - name: Build converter_service image
+        run: docker build -t gem-converter:ci ./converter_service

CHANGELOG.md

@@ -0,0 +1,76 @@
+# Changelog — fintech-microservices-core
+
+All notable changes follow [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+---
+
+## [Unreleased] — 2026-06-27
+
+### Added
+- **12-step production migration** — full transition from mock-data-driven UI to production-grade system
+- `src/server/operations.ts` — replaced in-memory Maps with SQLite-backed persistence via `better-sqlite3`
+- `src/server/ledger.ts` — real double-entry ledger replacing `getMockLedgerSnapshot()`
+- `src/server/config.ts` — APP_MODE gating (`mock` / `staging` / `production`) with hard production guards
+- `src/server/providers/card_provider.ts` — card issuance provider adapter (mock → live)
+- `src/server/providers/conversion_provider.ts` — fiat-to-BTC conversion provider adapter
+- `app/api/deposits/route.ts` — DB-wired deposit routes with ledger credit on creation
+- `app/api/cards/route.ts` — DB-wired card routes with provider provisioning
+- `app/api/compliance/route.ts` — compliance sweep proxy (mock + live mode)
+- `app/api/audit/route.ts` — immutable audit log proxy, admin-only
+- `app/page.tsx` — production React dashboard: all state from real API calls, no in-memory mocks
+- `tests/unit/test_compliance_engine.py` — GovernanceEngine unit tests
+- `tests/unit/test_audit_chain.py` — AuditTrailAgent chain integrity tests
+- `tests/unit/test_screening_rules.py` — RuleEngine unit tests
+- `tests/integration/test_api_routes.py` — card/fund route integration tests
+- `tests/security/test_webhook_signature.py` — HMAC security tests (timing attack resistance)
+- `tests/e2e/test_deposit_flow.py` — full deposit→ledger E2E flow
+- `Makefile` — developer CLI: install, lint, test, sweep, audit, docker, push
+- `docs/compliance-evidence.md` — partner-facing compliance evidence document
+- `docs/production-readiness.md` — production readiness checklist
+
+### Changed
+- `compliance_agents/screening/agent.py` — triple LLM cascade (OpenAI → Groq → Mistral) v3.0.0
+- `compliance_agents/shared/storage.py` — `llm_model`, `llm_prompt_version`, `has_llm_analysis` columns
+- `compliance_agents/audit_trail/agent.py` — quarantine model for corrupt entries
+
+### Fixed
+- Audit chain sequences 37–38: corrupt null-hash entries quarantined, chain re-stitched (54 valid + 2 quarantined)
+- `verify_chain_integrity` — handles corrupt rows without false broken-link reports
+- `_append` — anchors new entries to last valid hash, skips quarantined rows
+
+---
+
+## [0.3.0] — 2026-06-07
+
+### Added
+- Triple LLM provider cascade: OpenAI → Groq (llama-3.3-70b-versatile) → Mistral (mistral-small-latest)
+- Groq and Mistral API key support
+- Provider label stored in `screening_results.llm_model`
+
+### Changed
+- ScreeningAgent: primary provider remains OpenAI gpt-4o; Groq/Mistral as fallback
+- requirements.txt: added `groq>=0.4.0`
+
+---
+
+## [0.2.0] — 2026-06-06
+
+### Changed
+- Replaced Anthropic/Claude with OpenAI (gpt-4o) as LLM backend for ScreeningAgent
+- Updated `requirements.txt`: `openai>=1.30.0` replaces `anthropic`
+- Added `llm_model`, `llm_prompt_version`, `has_llm_analysis` to `screening_results` DB schema
+
+---
+
+## [0.1.0] — 2026-06-01
+
+### Added
+- Initial 4-agent compliance system: DataIngestion, Screening, AuditTrail, Reporting
+- GovernanceEngine with 16 controls across COBIT 2019, COSO ERM, GAO AI, IIA AI frameworks
+- Immutable audit chain with HMAC hash chaining
+- Daily governance sweep automation (07:00 WAT)
+- Foundry agent definitions (ComplianceAgent, SecurityAgent, LedgerAgent, PlatformAgent, BuildAgent)
+- Infrastructure: Terraform modules (VPC, EKS, RDS, ECR, S3, CloudWatch, WAF)
+- Kubernetes manifests, monitoring (Prometheus + Grafana + Alertmanager)
+- Docker Compose for local development

Makefile

@@ -0,0 +1,88 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# GEM ATR / fintech-microservices-core — Developer Makefile
+# ─────────────────────────────────────────────────────────────────────────────
+
+.PHONY: help install lint test test-unit test-integration test-security test-e2e \
+        sweep audit build docker-up docker-down push clean
+
+PYTHON        := python3
+PIP           := pip3
+PYTEST        := pytest
+APP_DIR       := apps/gem-atr-digital-easyway
+COMPLIANCE    := compliance_agents
+BRANCH        := production-infrastructure
+
+help:   ## Show this help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
+	  awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-22s\033[0m %s\n", $$1, $$2}'
+
+# ── Dependencies ──────────────────────────────────────────────────────────────
+
+install:   ## Install all Python dependencies
+	$(PIP) install -r $(COMPLIANCE)/requirements.txt -q
+	$(PIP) install pytest pytest-asyncio httpx -q
+	cd $(APP_DIR) && npm install
+
+# ── Code quality ──────────────────────────────────────────────────────────────
+
+lint:   ## Run linters (flake8 + tsc)
+	flake8 $(COMPLIANCE) --max-line-length=120 --ignore=E501,W503
+	cd $(APP_DIR) && npx tsc --noEmit
+
+# ── Tests ─────────────────────────────────────────────────────────────────────
+
+test:   ## Run all tests
+	$(PYTEST) tests/ -v --tb=short
+
+test-unit:   ## Run unit tests only
+	$(PYTEST) tests/unit/ -v --tb=short
+
+test-integration:   ## Run integration tests (requires services running)
+	$(PYTEST) tests/integration/ -v --tb=short
+
+test-security:   ## Run security tests
+	$(PYTEST) tests/security/ -v --tb=short
+
+test-e2e:   ## Run E2E tests (requires full stack on localhost)
+	$(PYTEST) tests/e2e/ -v --tb=short
+
+# ── Compliance ────────────────────────────────────────────────────────────────
+
+sweep:   ## Run a full governance sweep across all 16 controls
+	$(PYTHON) -c "from $(COMPLIANCE).governance.engine import GovernanceEngine; \
+	  r = GovernanceEngine().run_full_sweep(); \
+	  [print(f'  {fw.framework.value}: {sum(1 for c in fw.controls if c[\"status\"]==\"satisfied\")}/{len(fw.controls)}') for fw in r.frameworks]"
+
+audit:   ## Verify audit chain integrity
+	$(PYTHON) -c "from $(COMPLIANCE).audit_trail.agent import AuditTrailAgent; \
+	  r = AuditTrailAgent().verify_chain_integrity(); \
+	  print(f'Chain OK: {r[\"integrity_ok\"]} | Verified: {r[\"verified_entries\"]} | Quarantined: {r[\"quarantined_entries\"]}')"
+
+# ── Local development ─────────────────────────────────────────────────────────
+
+build:   ## Build Next.js app
+	cd $(APP_DIR) && npm run build
+
+dev:   ## Start Next.js dev server
+	cd $(APP_DIR) && npm run dev
+
+docker-up:   ## Start all services via Docker Compose
+	docker compose -f docker/docker-compose.yml up -d --build
+
+docker-down:   ## Stop all Docker services
+	docker compose -f docker/docker-compose.yml down
+
+# ── Git ───────────────────────────────────────────────────────────────────────
+
+push:   ## Commit and push to production-infrastructure branch
+	git add -A
+	git commit -m "chore: production build update $$(date +'%Y-%m-%d')"
+	git push origin $(BRANCH)
+
+# ── Clean ─────────────────────────────────────────────────────────────────────
+
+clean:   ## Remove build artifacts
+	find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
+	find . -name "*.pyc" -delete 2>/dev/null || true
+	rm -f gem-atr.db
+	cd $(APP_DIR) && rm -rf .next node_modules 2>/dev/null || true

apps/gem-atr-digital-easyway/.env.example

@@ -1,7 +1,40 @@
-NEXT_PUBLIC_MOCK_MODE=true
-CRON_SECRET=change-me-min-32-chars
-BANKING_WEBHOOK_SECRET=change-me-min-32-chars
-# Optional future:
-# CLERK_SECRET_KEY=
-# CLERK_PUBLISHABLE_KEY=
-# DATABASE_URL=
+# ─────────────────────────────────────────────────────────────────────────────
+# GEM ATR Digital — Environment Variables
+# Copy this file to .env.local for development.
+# Never commit .env.local or real secrets to version control.
+# ─────────────────────────────────────────────────────────────────────────────
+
+# ── App Mode ──────────────────────────────────────────────────────────────────
+# 'mock'        — no external calls, deterministic responses (default for local dev)
+# 'staging'     — uses real provider adapters with test credentials
+# 'production'  — full production mode; mock mode is blocked
+APP_MODE=mock
+
+# Legacy flag — keep false unless APP_MODE=mock
+NEXT_PUBLIC_MOCK_MODE=false
+
+# ── Database ──────────────────────────────────────────────────────────────────
+DATABASE_URL=./gem-atr.db
+
+# ── Security ──────────────────────────────────────────────────────────────────
+CRON_SECRET=change-me-cron-secret
+BANKING_WEBHOOK_SECRET=change-me-webhook-secret
+
+# ── Card Provider ─────────────────────────────────────────────────────────────
+CARD_PROVIDER_API_KEY=
+
+# ── Converter / Internal Service ──────────────────────────────────────────────
+CONVERTER_SERVICE_SECRET=change-me-converter-secret
+CONVERTER_SERVICE_URL=http://localhost:8003
+
+# ── Compliance & Audit Services ───────────────────────────────────────────────
+COMPLIANCE_SERVICE_URL=http://localhost:8001
+AUDIT_SERVICE_URL=http://localhost:8002
+
+# ── Observability ─────────────────────────────────────────────────────────────
+SENTRY_DSN=
+LOG_LEVEL=info
+
+# ── Mock user override (APP_MODE=mock only) ───────────────────────────────────
+# Set to 'admin' to see admin panels in the dashboard
+MOCK_USER_ROLE=user

apps/gem-atr-digital-easyway/app/api/audit/route.ts

@@ -0,0 +1,49 @@
+/**
+ * /api/audit — Proxy to audit_service; admin-only.
+ * Returns the most recent immutable audit log entries.
+ */
+
+import { NextRequest, NextResponse } from 'next/server';
+import { requireAdmin } from '@/src/server/auth';
+import { getConfig } from '@/src/server/config';
+
+export const dynamic = 'force-dynamic';
+
+export async function GET(req: NextRequest) {
+  try {
+    requireAdmin();
+    const { auditServiceUrl, appMode } = getConfig();
+    const limit = Number(req.nextUrl.searchParams.get('limit') ?? '25');
+
+    if (appMode === 'mock') {
+      const now = new Date();
+      return NextResponse.json({
+        mode: 'mock',
+        entries: Array.from({ length: Math.min(limit, 10) }, (_, i) => ({
+          sequence_number: 57 - i,
+          event_type: i === 0 ? 'system_event' : i === 3 ? 'sar_generated' : 'compliance_check',
+          source_agent: ['GovernanceEngine', 'ReportingAgent', 'ScreeningAgent'][i % 3],
+          timestamp: new Date(now.getTime() - i * 3_600_000).toISOString(),
+          chain_valid: true,
+        })),
+        chain_integrity: true,
+        verified_entries: 54,
+        quarantined_entries: 2,
+      });
+    }
+
+    const res = await fetch(`${auditServiceUrl}/entries?limit=${limit}`, {
+      headers: { 'X-Admin': 'true' },
+    });
+
+    if (!res.ok) {
+      return NextResponse.json({ error: 'Audit service unavailable' }, { status: 502 });
+    }
+
+    return NextResponse.json(await res.json());
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Internal error';
+    const status = message.includes('role') || message.includes('Admin') ? 403 : 500;
+    return NextResponse.json({ error: message }, { status });
+  }
+}