feat: end-to-end WalStream CDC platform with durable ingest/replay, FastAPI control plane, Next.js dashboard, and observability

Author: Vaibtan

.claude/settings.local.json

@@ -5,7 +5,26 @@
       "Bash(tree:*)",
       "Bash(mkdir:*)",
       "WebSearch",
-      "Bash(findstr:*)"
+      "Bash(findstr:*)",
+      "Bash(find:*)",
+      "Bash(npx shadcn@latest add:*)",
+      "Bash(npm install:*)",
+      "Bash(npm run build:*)",
+      "Bash(test:*)",
+      "Bash(grep:*)",
+      "Bash(python -c:*)",
+      "Bash(python -m pytest:*)",
+      "Bash(.venv/Scripts/python -m pytest control/tests/ -x -v --tb=short)",
+      "Bash(.venv/Scripts/pip list:*)",
+      "Bash(.venv/Scripts/python -m pip:*)",
+      "Bash(.venv/Scripts/python.exe -m pip:*)",
+      "Bash(.venv/Scripts/python.exe:*)",
+      "Bash(where:*)",
+      "Bash(ls:*)"
     ]
-  }
+  },
+  "enableAllProjectMcpServers": true,
+  "enabledMcpjsonServers": [
+    "context7"
+  ]
 }

.env.example

@@ -1,75 +1,32 @@
-# WalStream CDC Configuration
-# Copy this file to .env and adjust values as needed
-
-# ==========================================================================
-# Database
-# ==========================================================================
-
 # Source database (PostgreSQL with WAL)
 POSTGRES_HOST=localhost
 POSTGRES_USER=repluser
 POSTGRES_PASSWORD=replpass
 POSTGRES_DB=walstreamdb
-
 # Control plane database
 DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5433/walstream_control
-
-# ==========================================================================
-# Message Queues
-# ==========================================================================
-
-# Redis
 REDIS_URL=redis://localhost:6379/0
 REDIS_STREAM=walstream:events
 REDIS_STREAM_MAXLEN=100000
-
 # Kafka
 KAFKA_BROKER=localhost:29092
 KAFKA_TOPIC=walstream.archive
-
-# ==========================================================================
-# Services
-# ==========================================================================
-
 # Replayer gRPC
 REPLAYER_HOST=localhost
 REPLAYER_PORT=50051
 REPLAYER_TIMEOUT_SECONDS=30
-
 # Target database for replay
 TARGET_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/walstreamdb
-
 # Dedup database
 DEDUP_DATABASE_URL=postgresql://postgres:postgres@localhost:5433/walstream_control
-
-# ==========================================================================
-# Security
-# ==========================================================================
-
-# JWT Secret (change in production!)
 SECRET_KEY=change-me-in-production-use-long-random-string
 JWT_ALGORITHM=HS256
 JWT_EXPIRATION_MINUTES=60
-
-# ==========================================================================
-# CORS
-# ==========================================================================
-
 CORS_ORIGINS=["http://localhost:3000"]
-
-# ==========================================================================
-# Worker Configuration
-# ==========================================================================
-
 JOB_WORKER_COUNT=4
 JOB_POLL_INTERVAL_SECONDS=1.0
 JOB_HEARTBEAT_INTERVAL_SECONDS=10.0
 JOB_LEASE_DURATION_SECONDS=300
-
-# ==========================================================================
-# Observability
-# ==========================================================================
-
 METRICS_PORT=9091
 LOG_LEVEL=INFO
 DEBUG=false

.mcp.json

@@ -0,0 +1,11 @@
+{
+  "mcpServers": {
+    "context7": {
+      "type": "http",
+      "url": "https://mcp.context7.com/mcp",
+      "headers": {
+        "CONTEXT7_API_KEY": "ctx7sk-6467cf35-65f9-4546-8719-4e820ec50818"
+      }
+    }
+  }
+}

Makefile

@@ -0,0 +1,85 @@
+.PHONY: up down logs test lint proto migrate migrate-check migrate-new clean help
+
+up:
+	docker-compose up -d
+down:
+	docker-compose down
+## Tail logs for all services
+logs:
+	docker-compose logs -f --tail=50
+
+## Start monitoring stack (Prometheus + Grafana + Alertmanager)
+up-monitoring:
+	docker-compose --profile monitoring up -d
+## Run Alembic migrations to head
+migrate:
+	cd control && alembic upgrade head
+
+## Check current DB revision
+migrate-check:
+	cd control && alembic current
+
+## Create a new migration (usage: make migrate-new MSG="add foo column")
+migrate-new:
+	cd control && alembic revision --autogenerate -m "$(MSG)"
+## Show migration history
+migrate-history:
+	cd control && alembic history --verbose
+## Regenerate protobuf Python files
+proto:
+	cd walstream-proto && python -m walstream_proto.generate
+
+test:
+	pytest --tb=short -q
+
+test-cov:
+	pytest --cov=control --cov=ingestor --cov=replayer --tb=short
+
+## Run proto compatibility tests
+test-proto:
+	pytest walstream-proto/tests/ --tb=short -q
+
+## Run delivery-semantics focused tests
+test-delivery:
+	pytest -k "delivery or dedup or idempoten" --tb=short -q
+
+lint:
+	ruff check .
+
+fmt:
+	ruff format .
+
+typecheck:
+	mypy control/ ingestor/ replayer/
+
+check: lint typecheck
+
+dev-control:
+	cd control && uvicorn app.main:app --reload --port 8000
+
+dev-ingestor:
+	python ingestor/ingestor.py
+
+dev-replayer:
+	python replayer/server.py
+
+dev-frontend:
+	cd frontend && npm run dev
+
+## Remove Python caches and build artifacts
+clean:
+	find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
+	find . -type d -name .pytest_cache -exec rm -rf {} + 2>/dev/null || true
+	find . -type d -name .mypy_cache -exec rm -rf {} + 2>/dev/null || true
+	find . -name "*.pyc" -delete 2>/dev/null || true
+
+## Remove Docker volumes (DESTRUCTIVE)
+clean-volumes:
+	docker-compose down -v
+
+help:
+	@echo "WalStream CDC - Available targets:"
+	@echo ""
+	@grep -E '^## ' $(MAKEFILE_LIST) | sed 's/^## /  /'
+	@echo ""
+	@echo "Run 'make <target>' to execute."