diff --git a/database_worker/Dockerfile b/database_worker/Dockerfile new file mode 100644 index 00000000..781cbb57 --- /dev/null +++ b/database_worker/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.13-slim + +RUN apt-get update && \ + apt-get install -y --no-install-recommends postgresql-client zstd && \ + rm -rf /var/lib/apt/lists/* + +COPY database_worker/server.py /app/server.py + +WORKDIR /app +EXPOSE 8002 + +CMD ["python", "server.py"] diff --git a/database_worker/railway.md b/database_worker/railway.md new file mode 100644 index 00000000..622ea2f8 --- /dev/null +++ b/database_worker/railway.md @@ -0,0 +1,45 @@ +# Railway Configuration for `database_worker` + +Deploy database_worker (PlanExe database maintenance service) to Railway as an internal HTTP service. + +This service provides database backup (via `pg_dump`) and is called by `frontend_multi_user`. It should **not** be exposed publicly. + +## Service variables example + +``` +PGHOST="${{shared.PLANEXE_POSTGRES_HOST}}" +PGPORT="5432" +PGDATABASE="planexe" +PGUSER="planexe" +PGPASSWORD="${{shared.PLANEXE_POSTGRES_PASSWORD}}" +PLANEXE_DATABASE_WORKER_API_KEY="${{shared.PLANEXE_DATABASE_WORKER_API_KEY}}" +``` + +## Required Environment Variables + +- `PGHOST` — Postgres host. On Railway, use the internal hostname (e.g. `postgres.railway.internal`). The Docker Compose default `database_postgres` does not resolve on Railway. +- `PGPASSWORD` — Postgres password. + +## Optional Environment Variables + +- `PLANEXE_DATABASE_WORKER_API_KEY` — If set, the `/backup` endpoint requires this key in the `X-Database-Worker-Key` header. Should match the same variable on `frontend_multi_user`. +- `PLANEXE_DATABASE_WORKER_PORT` — Port to listen on (default: `8002`). On Railway, the auto-injected `PORT` is not used since this is an internal service. + +## Networking + +This service is **internal only** — do not assign a public domain. The `frontend_multi_user` service calls it via Railway's private networking: + +``` +PLANEXE_DATABASE_WORKER_URL="http://databaseworker.railway.internal:8002" +``` + +Set this variable on the `frontend_multi_user` service. + +## Endpoints + +- `GET /healthcheck` — returns `ok` (used by Railway health checks) +- `GET /backup` — streams a gzipped `pg_dump` of the database. Protected by `PLANEXE_DATABASE_WORKER_API_KEY` if configured. + +## Volume — None + +The service is stateless. Backups are streamed directly to the caller without writing to disk. diff --git a/database_worker/railway.toml b/database_worker/railway.toml new file mode 100644 index 00000000..e8c8d7b0 --- /dev/null +++ b/database_worker/railway.toml @@ -0,0 +1,11 @@ +[build] +builder = "DOCKERFILE" +dockerfilePath = "/database_worker/Dockerfile" +watchPatterns = ["/database_worker/**"] +context = "." + +[deploy] +healthcheckPath = "/healthcheck" +healthcheckTimeout = 100 +restartPolicyType = "ON_FAILURE" +restartPolicyMaxRetries = 10 diff --git a/database_worker/server.py b/database_worker/server.py new file mode 100644 index 00000000..96b22265 --- /dev/null +++ b/database_worker/server.py @@ -0,0 +1,119 @@ +"""Minimal HTTP server that streams pg_dump output as a compressed download.""" +import os +import shutil +import subprocess +import logging +from datetime import datetime, UTC +from http.server import HTTPServer, BaseHTTPRequestHandler + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger(__name__) + +PGHOST = os.environ.get("PGHOST", "database_postgres") +PGPORT = os.environ.get("PGPORT", "5432") +PGDATABASE = os.environ.get("PGDATABASE", "planexe") +PGUSER = os.environ.get("PGUSER", "planexe") +PGPASSWORD = os.environ.get("PGPASSWORD", "planexe") +API_KEY = os.environ.get("PLANEXE_DATABASE_WORKER_API_KEY", "") +PORT = int(os.environ.get("PLANEXE_DATABASE_WORKER_PORT", "8002")) + +# zstd typically compresses better and faster than gzip, so it's preferred. +# pg_dump >= 16 supports -Z zstd natively; also requires the zstd binary. +_HAS_ZSTD = False +try: + version_output = subprocess.check_output(["pg_dump", "--version"], text=True) + pg_major = int(version_output.strip().split()[-1].split(".")[0]) + if pg_major >= 16 and shutil.which("zstd"): + _HAS_ZSTD = True +except Exception: + pass +logger.info("Compression: %s (pg_dump %s)", "zstd" if _HAS_ZSTD else "gzip", version_output.strip() if 'version_output' in dir() else "unknown") + + +class BackupHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/healthcheck": + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"ok") + return + + if self.path != "/backup": + self.send_error(404) + return + + # Simple API key auth if configured + if API_KEY: + auth = self.headers.get("X-Database-Worker-Key", "") + if auth != API_KEY: + self.send_error(403, "Invalid backup API key") + return + + if _HAS_ZSTD: + compress_flag = "zstd:6" + ext = "sql.zst" + content_type = "application/zstd" + else: + compress_flag = "6" + ext = "sql.gz" + content_type = "application/gzip" + + timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") + filename = f"{timestamp}_planexe_backup.{ext}" + + logger.info("Starting database backup: %s (%s)", filename, "zstd" if _HAS_ZSTD else "gzip") + + env = os.environ.copy() + env["PGPASSWORD"] = PGPASSWORD + + proc = subprocess.Popen( + [ + "pg_dump", + "-h", PGHOST, + "-p", PGPORT, + "-U", PGUSER, + "-d", PGDATABASE, + "--no-owner", + "--no-privileges", + "-Z", compress_flag, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Disposition", f'attachment; filename="{filename}"') + self.end_headers() + + try: + while True: + chunk = proc.stdout.read(256 * 1024) + if not chunk: + break + self.wfile.write(chunk) + + proc.wait() + if proc.returncode != 0: + stderr = proc.stderr.read().decode("utf-8", errors="replace") + logger.error("pg_dump failed (rc=%d): %s", proc.returncode, stderr) + else: + logger.info("Backup complete: %s", filename) + except BrokenPipeError: + logger.warning("Client disconnected during backup") + proc.kill() + finally: + proc.stdout.close() + proc.stderr.close() + + def log_message(self, format, *args): + if "/healthcheck" not in (args[0] if args else ""): + logger.info(format, *args) + + +if __name__ == "__main__": + server = HTTPServer(("0.0.0.0", PORT), BackupHandler) + logger.info("Backup server listening on port %d", PORT) + server.serve_forever() diff --git a/docker-compose.yml b/docker-compose.yml index 2bd9d77e..35196f33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -197,6 +197,8 @@ services: PLANEXE_FRONTEND_MULTIUSER_DB_PASSWORD: ${PLANEXE_POSTGRES_PASSWORD:-planexe} PLANEXE_FRONTEND_MULTIUSER_ADMIN_USERNAME: ${PLANEXE_FRONTEND_MULTIUSER_ADMIN_USERNAME:-admin} PLANEXE_FRONTEND_MULTIUSER_ADMIN_PASSWORD: ${PLANEXE_FRONTEND_MULTIUSER_ADMIN_PASSWORD:-admin} + PLANEXE_DATABASE_WORKER_URL: ${PLANEXE_DATABASE_WORKER_URL:-http://database_worker:8002} + PLANEXE_DATABASE_WORKER_API_KEY: ${PLANEXE_DATABASE_WORKER_API_KEY:-} ports: - "${PLANEXE_FRONTEND_MULTIUSER_PORT:-5001}:5000" volumes: @@ -213,6 +215,23 @@ services: # instead of restart-looping. Runtime crashes (exit != 0) still restart. restart: on-failure + database_worker: + build: + context: . + dockerfile: database_worker/Dockerfile + container_name: database_worker + depends_on: + database_postgres: + condition: service_healthy + environment: + PGHOST: database_postgres + PGPORT: "5432" + PGDATABASE: ${PLANEXE_POSTGRES_DB:-planexe} + PGUSER: ${PLANEXE_POSTGRES_USER:-planexe} + PGPASSWORD: ${PLANEXE_POSTGRES_PASSWORD:-planexe} + PLANEXE_DATABASE_WORKER_API_KEY: ${PLANEXE_DATABASE_WORKER_API_KEY:-} + restart: unless-stopped + mcp_cloud: build: context: . diff --git a/frontend_multi_user/src/app.py b/frontend_multi_user/src/app.py index 8dbee3c7..a632f709 100644 --- a/frontend_multi_user/src/app.py +++ b/frontend_multi_user/src/app.py @@ -2083,6 +2083,17 @@ def _vacuum_task_item(self) -> dict[str, Any]: result["error"] = str(e) return result + def _proxy_backup_response(self) -> requests.Response: + """Start a streaming GET to the database_worker backup endpoint.""" + worker_url = os.environ.get("PLANEXE_DATABASE_WORKER_URL", "http://database_worker:8002") + api_key = os.environ.get("PLANEXE_DATABASE_WORKER_API_KEY", "") + headers = {} + if api_key: + headers["X-Database-Worker-Key"] = api_key + resp = requests.get(f"{worker_url}/backup", headers=headers, stream=True, timeout=600) + resp.raise_for_status() + return resp + def _build_reconciliation_report(self, max_tasks: int, tolerance_usd: float) -> tuple[list[dict[str, Any]], dict[str, Any]]: tasks = ( PlanItem.query @@ -3078,6 +3089,24 @@ def admin_database(): vacuum_result=vacuum_result, ) + @self.app.route('/admin/database/backup') + @admin_required + def admin_database_backup(): + try: + upstream = self._proxy_backup_response() + return Response( + upstream.iter_content(chunk_size=256 * 1024), + mimetype=upstream.headers.get('Content-Type', 'application/octet-stream'), + headers={ + 'Content-Disposition': upstream.headers.get( + 'Content-Disposition', 'attachment; filename="planexe_backup.sql.gz"' + ), + }, + ) + except Exception as e: + logger.exception("Failed to proxy database backup") + return jsonify({"error": str(e)}), 502 + @self.app.route('/ping/stream') @login_required def ping_stream(): diff --git a/frontend_multi_user/templates/admin/database.html b/frontend_multi_user/templates/admin/database.html index 532fe6ec..4e42d44e 100644 --- a/frontend_multi_user/templates/admin/database.html +++ b/frontend_multi_user/templates/admin/database.html @@ -123,6 +123,21 @@ .btn-purge:hover { background: #a93226; } + .btn-backup { + background: #27ae60; + color: #fff; + border: none; + padding: 0.6rem 1.5rem; + border-radius: 4px; + font-size: 0.95rem; + cursor: pointer; + text-decoration: none; + display: inline-block; + } + .btn-backup:hover { + background: #219a52; + color: #fff; + } {% endblock %} @@ -175,6 +190,14 @@
+ Download a compressed snapshot of all database tables (COPY TO format, gzipped).
+