diff --git a/backend/Dockerfile b/backend/Dockerfile index b32339a..4a94b9a 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -17,6 +17,18 @@ COPY backend ./backend RUN npm run build --workspace=@wc-telegram/backend +FROM dependencies AS migration + +COPY backend ./backend +COPY scripts/ops/run-migrations.sh /usr/local/bin/wctm-run-migrations + +RUN chmod 0555 /usr/local/bin/wctm-run-migrations + +WORKDIR /app +USER node + +ENTRYPOINT ["/usr/local/bin/wctm-run-migrations"] + FROM node:24.20.0-alpine3.24@sha256:e67514e5d0f6c46656005e1b693b2ec9d52e80b641307de684d4a015ba7a4eaf AS production ARG ALPINE_OPENSSL_VERSION=3.5.8-r0 diff --git a/docker-compose.yml b/docker-compose.yml index 19dc849..36c3eb8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,6 +83,24 @@ services: networks: - wc-telegram-net + migrate: + profiles: ['operations'] + build: + context: . + dockerfile: backend/Dockerfile + target: migration + init: true + read_only: true + tmpfs: + - /tmp:size=16m,mode=1777 + security_opt: + - no-new-privileges:true + depends_on: + postgres: + condition: service_healthy + networks: + - wc-telegram-net + telegram-bot: build: context: . diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index f93b1aa..b9c6ffc 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -1173,4 +1173,92 @@ the staged D-030 architecture or start P7.2+. --- -Next decision number: D-031. +## D-031 + +Date + +2026-09-16 + +Decision + +P7.2 separates production schema migration from normal application runtime. +The backend continues to authenticate only as restricted `wctm_runtime` and +never runs a migration during startup. An independently invokable, non-root, +read-only Compose operations job contains the exact checked-out Prisma CLI, +schema, and migrations and accepts a distinct existing owner/migration +credential only for that invocation through a protected external file. Its +entrypoint rejects `wctm_runtime`, Prisma Migrate deploy remains authoritative, +and the ephemeral container is removed after success or failure. + +The supported deployment sequence is revision-locked and fail-fast: clean and +synchronized repository, production configuration audit, verified P7.3 backup, +image build, explicit migration before application cutover, backend/bot +recreation, health/readiness, restricted-role verification, and bounded +functional smoke. PostgreSQL and Redis are reconciled separately onto the +reviewed immutable references only after backup and exact named-volume +verification; their volumes are preserved and never deleted. Schema migrations +are not assumed reversible, and database recovery delegates to P7.3. + +Reason + +Keeping migration authority out of long-lived runtime preserves P7.1 least +privilege while providing one deterministic, auditable production path without +a new deployment platform or automatic schema mutation. + +Status + +Accepted for repository implementation. Production acceptance remains pending +A-owned live validation. + +--- + +## D-032 + +Date + +2026-09-16 + +Decision + +P7.3 uses timestamped PostgreSQL custom-format backups with restrictive local +permissions, atomic finalization, dump-list readability validation, SHA-256, +and non-secret metadata. Retention is explicit, count-based, defaults to 14 +valid daily sets, never removes the newest valid set, and targets only the exact +WCTM backup naming pattern. A provider-neutral executable hook supplies +off-host transfer. The repository rclone implementation requires the remote +dump content to match the locally generated SHA-256: it uses a compatible +native remote SHA-256 when available, otherwise streams the remote dump through +rclone and hashes it locally without replacing the local backup. Remote size +remains a secondary check for all three transferred artifacts. Copy, metadata, +size, content-integrity, or integrity-verification failures exit non-zero. + +Daily scheduling uses repository-controlled systemd service/timer templates +whose external environment file contains paths and an optional remote name but +no committed credential. Restore validation always targets a uniquely generated +PostgreSQL 16.15 container and volume with no network or published port; it has +no production URL/overwrite option, verifies checksum and dump readability, +restores migration/schema/data state, performs configurable critical row-count +checks, and removes only its own isolated resources. + +Database recovery also requires the matching repository revision and valid APP +encryption key. A previous APP key is relevant only to a backup taken during a +controlled historical rotation window. JWT, callback, internal service, bot, +and database credentials restore their own trust boundaries and do not decrypt +stored ciphertext. Initial targets are a 24-hour RPO and eight-hour RTO, not +contractual guarantees or an SLA. + +Reason + +The P7.1 one-off restore proof did not provide repeatable backup, retention, +off-host, scheduling, guarded restore, or disaster-recovery operations. This +bounded repository mechanism supplies those controls without a backup catalog, +new application table, public endpoint, provider lock-in, or multi-region HA. + +Status + +Accepted for repository implementation. Production scheduling, destination, +restore exercise, and milestone acceptance remain pending A-owned validation. + +--- + +Next decision number: D-033. diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 41100f9..fa23d2c 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -2,7 +2,7 @@ **Generated:** 2026-07-19 -**Updated:** 2026-09-06 +**Updated:** 2026-09-16 **Reason:** Maintaining implementation continuity @@ -43,7 +43,7 @@ n8n is **NOT** part of the production architecture (D-008, prototype only). --- -## 3. Architectural Decisions (D-001–D-030) +## 3. Architectural Decisions (D-001–D-032) | ID | Decision | Status | | ----- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | @@ -77,8 +77,10 @@ n8n is **NOT** part of the production architecture (D-008, prototype only). | D-028 | One backend-authoritative Tenant ACTIVE/SUSPENDED lifecycle with derived expiry, operator-only mutation, explicit product gates, continued projections, terminal notification suppression, and no commercial billing scope | Accepted | | D-029 | Phase 7 production readiness precedes deferred Phase 6; P7.1 establishes the bounded repository security baseline while host changes, credential rotation, deployment, and P7.2+ remain separately controlled | Accepted | | D-030 | P7.1 uses a temporary decrypt-only previous key, current-key-only writes, and a trusted-shell row-transactional rotation command for the exact five encrypted database fields | Accepted | +| D-031 | P7.2 separates restricted backend runtime from an ephemeral explicit Prisma migration identity and defines revision-locked fail-fast deployment plus volume-preserving immutable data-service reconciliation | Accepted | +| D-032 | P7.3 establishes atomic verified PostgreSQL backups, explicit retention, verified provider-neutral off-host copy, daily systemd scheduling, isolated guarded restore, DR dependencies, and target-only RPO/RTO | Accepted | -Next decision number: **D-031**, if a future task produces a genuine +Next decision number: **D-033**, if a future task produces a genuine architectural or product decision. --- @@ -1179,9 +1181,62 @@ Dependency and automated evidence: missing or excessive privileges across all 20 application tables, while the existing privileged owner remains available for Prisma migration execution. -This closure branch changes documentation only. It performs no VPS, production, -secret, database, deployment, schema, migration, application, product, P7.2+, -or Phase 6 action. +#### P7.2 — Production Migration & Deployment Path (repository complete; awaiting validation) + +- Implementation commit: + `91e16fd5933ffa4e6386d649f21f9e2592833ed5 chore(ops): establish production +migration and deployment path`. +- D-031 keeps `wctm_runtime` as the backend-only restricted identity and moves + Prisma tooling into an explicit ephemeral Compose operations target. The + wrapper reads one protected external migration URL, rejects `wctm_runtime`, + runs exact checked-out migrations, removes the container, and never exposes + that credential to normal backend runtime. +- `deploy-production.sh` is revision-locked and fail-fast across clean/synced + Git, Compose, config audit, fresh verified P7.3 backup, image build, explicit + migration before cutover, backend/bot recreation, health/readiness, and + restricted-role verification. Backend startup remains `node dist/main.js` + with no automatic schema mutation or `db push`. +- `reconcile-data-images.sh` requires the verified backup plus explicit current + PostgreSQL/Redis volume names, preserves those volumes, recreates only those + services on the P7.1 immutable references, verifies health, migration/key + continuity and exact image configuration, and leaves apps stopped for cutover. +- Failure guidance distinguishes safe application-image restoration from + irreversible/forward-only schema effects and hands database recovery to P7.3. + Repository implementation is ready for B review; production acceptance is + pending A's combined runbook. + +#### P7.3 — Backup, Restore & Disaster Recovery (repository complete; awaiting validation) + +- Implementation commit: + `ddde3c705b249a29a131a3bfc96763f17b6e9cbc chore(ops): add production backup +and recovery path`. +- D-032 adds atomic mode-0600 custom-format dumps with UTC unique names, + `pg_restore --list`, SHA-256, and non-secret revision/migration/size/version + metadata. Explicit retention defaults to 14 valid sets, always protects the + newest, and ignores unrelated/incomplete/corrupt files. +- A provider-neutral executable hook is supported; the supplied rclone hook + copies dump/checksum/metadata, verifies their remote sizes, and requires the + remote dump to match the local SHA-256 through a compatible native remote + hash or a locally hashed rclone stream. Failure is non-zero. + Daily Ubuntu systemd templates load paths/remote name from an external file, + expose failures through service status/journal, and contain no credential. +- Restore accepts one explicit local backup only and creates a generated + PostgreSQL 16.15 container/volume with `--network none`. It verifies checksum, + dump readability, exact repository migration count, public schema and chosen + critical table rows, and removes only its own isolated resources. +- DR covers application/container loss, failed deployment, database corruption, + and new-host recovery. Stored ciphertext requires the matching APP key; other + service/signing/database credentials do not decrypt it. Initial targets are a + 24-hour RPO and eight-hour RTO, not SLA guarantees. +- The executable combined A session is + `docs/operations/P7_2_P7_3_RUNBOOK.md`: preflight; fresh verified/off-host + backup; migration identity proof; immutable PostgreSQL/Redis reconciliation; + app deployment; bounded smoke; post-deployment backup; isolated restore; and + final permission/migration/backup/secret/health scans. + +This repository cycle performs no VPS, production, secret, live database, +deployment, schema, migration, or product mutation. P7.2 and P7.3 are not +production-complete. P7.4+, Phase 6, and product work remain unstarted. ## 5. Current Repository Structure @@ -1190,15 +1245,15 @@ NestJS API, `telegram-bot/` for the grammY process, and `wp-content/plugins/` fo the lightweight connector. The larger `apps/`, `packages/`, and `infrastructure/` layout remains a planned target rather than current structure. -Current branch: `docs/p7.1-production-security-closure`. +Current branch: `chore/p7.2-p7.3-production-operations`. --- ## 6. Current Blockers No open Phase-5 or P7.1 blocker remains. Public launch remains gated by -unstarted P7.2–P7.8. Historical provenance remains unresolved and assigned to -P7.8. Generalized backup/restore/DR remains P7.3, monitoring P7.4, DATE-001 +P7.2/P7.3 production acceptance and unstarted P7.4–P7.8. Historical provenance +remains unresolved and assigned to P7.8. Monitoring remains P7.4, DATE-001 P7.5, network/runtime resilience P7.6, and AuditLog structural immutability P7.7. @@ -1206,9 +1261,10 @@ P7.7. ## 7. Current Task -P7.1 — Production Security Baseline is complete. P7.2 — Production Migration & -Deployment Path is next but unstarted pending separate approval. Do not start -P7.2+, Phase 6, or product expansion from this docs-only closure. +P7.1 is complete. P7.2 and P7.3 repository implementation is complete and ready +for B review; both await A-owned live validation and neither is production- +complete. P7.4 is next only after both acceptances and remains unstarted. Do not +start P7.4+, Phase 6, or product expansion. ### Last completed product milestone: M22 diff --git a/docs/MASTER-ROADMAP.md b/docs/MASTER-ROADMAP.md index 74fc119..d141878 100644 --- a/docs/MASTER-ROADMAP.md +++ b/docs/MASTER-ROADMAP.md @@ -294,17 +294,23 @@ commercial/product expansion. Milestones, in order: - P7.1 — Production Security Baseline — ✅ Complete -- P7.2 — Production Migration & Deployment Path — ⬜ Next / unstarted -- P7.3 — Backup, Restore & Disaster Recovery +- P7.2 — Production Migration & Deployment Path — 🟡 Repository implemented; + awaiting B review and A-owned production validation +- P7.3 — Backup, Restore & Disaster Recovery — 🟡 Repository implemented; + awaiting B review and A-owned production validation - P7.4 — Monitoring & Alerting - P7.5 — Data & Time Correctness - P7.6 — Network & Runtime Reliability - P7.7 — Audit & Operational Integrity - P7.8 — Final Launch Readiness Gate -P7.1 is complete following accepted production validation. P7.2 is the next -allowed Phase 7 milestone and remains unstarted. P7.3–P7.8 and Phase 6 remain -unstarted; Phase 7 is not complete. +P7.1 is complete following accepted production validation. P7.2 and P7.3 now +have repository-side implementation for explicit least-privilege migration, +deterministic deployment, immutable database/cache image reconciliation, +verified backup/off-host retention/scheduling, isolated restore, and disaster +recovery. Both remain open pending B review and A-owned live validation; neither +is production-complete. P7.4 is next only after both are accepted. P7.4–P7.8 +and Phase 6 remain unstarted; Phase 7 is not complete. --- diff --git a/docs/PHASE_BREAKDOWN.md b/docs/PHASE_BREAKDOWN.md index 40d20f1..e3e725a 100644 --- a/docs/PHASE_BREAKDOWN.md +++ b/docs/PHASE_BREAKDOWN.md @@ -758,7 +758,40 @@ Node 24.20.0 now backs both application images; Node, PostgreSQL 16, and Redis 7 use exact patch/distro tags plus immutable digests. The initial overprivileged production identity was replaced for backend runtime by the validated restricted role. Historical release-provenance findings remain without history -rewrite and are still assigned to P7.8. P7.2 is next but unstarted; P7.3 through +rewrite and are still assigned to P7.8. + +### P7.2 — Production Migration & Deployment Path 🟡 Repository Complete / Awaiting Validation + +- D-031 establishes an ephemeral, non-root Compose migration job containing the + exact checked-out Prisma migration state. A distinct protected migration URL + is supplied only for the operation; `wctm_runtime` is rejected and remains + the backend-only restricted identity with no migration-table or DDL access. +- Revision-locked deployment fails before cutover on preflight, config audit, + backup, build, or migration failure. Backend startup performs no migration or + `db push`; health/readiness and restricted-role verification follow cutover. +- Guarded PostgreSQL/Redis reconciliation requires a verified backup and exact + named-volume confirmation, preserves volumes, recreates only those services + on the reviewed immutable references, and verifies health/data continuity. +- Repository implementation is ready for B review. Production validation and + A acceptance remain required; P7.2 is not production-complete. + +### P7.3 — Backup, Restore & Disaster Recovery 🟡 Repository Complete / Awaiting Validation + +- D-032 establishes atomic PostgreSQL custom-format backups, SHA-256 and + readability verification, non-secret metadata, explicit 14-valid-set default + retention, verified provider-neutral off-host transfer, and daily systemd + templates with no committed credentials. +- Guarded restore always targets an isolated PostgreSQL 16.15 container/volume + with no network exposure, verifies repository migration state, schema and + configurable critical row counts, and cannot target production by default. +- The DR runbook covers application/container, deployment, database, and total + VPS loss; distinguishes APP-key decryption dependency from other service + credentials; and sets target-only 24-hour RPO/eight-hour RTO objectives. +- Repository implementation is ready for B review. A must choose the off-host + destination, install/enable scheduling, and execute the combined live backup, + deployment and isolated-restore runbook. P7.3 is not production-complete. + +P7.4 remains next only after P7.2/P7.3 production acceptance. P7.4 through P7.8 are unstarted. Phase 7 is not complete. ### Approved Phase 7 order diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index e5048f8..bf91d82 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -15,18 +15,21 @@ M1–M16 remain complete and unchanged. By explicit A decision D-029, the curren phase is Phase 7 — Production Readiness, which executes before Phase 6. P7.1 — Production Security Baseline is complete after successful A-owned production validation. Phase 7 is not complete. P7.2 — Production Migration & Deployment -Path is next but unstarted. Phase 6 commercial SaaS work remains deferred and -unstarted until Phase 7 completes and A separately authorizes it. Unrestricted -public launch is not approved. +Path and P7.3 — Backup, Restore & Disaster Recovery are repository-implemented +and ready for B review, but remain open pending A-owned production validation. +P7.4 is next only after both are accepted. Phase 6 commercial SaaS work remains +deferred and unstarted until Phase 7 completes and A separately authorizes it. +Unrestricted public launch is not approved. --- Current Task -P7.1 — Production Security Baseline is complete. P7.2 — Production Migration & -Deployment Path is the next allowed Phase 7 work and remains unstarted pending -separate approval. No product feature work is active. Do not begin P7.2, P7.3+, -Phase 6, or product expansion without separate approval. +P7.1 — Production Security Baseline is complete. P7.2 and P7.3 repository work +is implemented in one approved operations cycle and awaits B review plus A-owned +live validation. Neither is production-complete. P7.4 remains unstarted and is +next only after both acceptances. No product feature work is active. Do not +begin P7.4+, Phase 6, or product expansion without separate approval. --- @@ -38,7 +41,7 @@ Project Version Repository -Current branch: `docs/p7.1-production-security-closure`. +Current branch: `chore/p7.2-p7.3-production-operations`. M22 implementation commit: `35f9e72335c8ed0c6a039497d92bed763dc68fb5 feat(entitlements): add MVP tenant @@ -778,7 +781,7 @@ WooCommerce Webhooks Current Branch -fix/p7.1-production-secret-rotation +chore/p7.2-p7.3-production-operations --- @@ -794,10 +797,9 @@ dependent sorting or reporting may be affected. AuditLog structural immutability is not yet enforced; the schema includes an updatable timestamp. A future approved decision must define enforcement. -The production backend image omits the Prisma CLI because production dependency -installation excludes the Prisma development dependency. M3 migrations were -verified with the Docker builder stage; the production migration execution path -requires a future approved infrastructure correction. +The normal production backend image intentionally omits the Prisma CLI. P7.2 +adds a separate ephemeral operations image with exact Prisma tooling and +migration state; live validation of that path remains pending A. --- @@ -809,26 +811,25 @@ AuditLog immutability enforcement is deferred to a future approved task. Current Blockers -No open Phase-5 or P7.1 blocker remains. Public launch remains gated by the -unstarted P7.2–P7.8 milestones. Four historical implementation-provenance -findings remain unresolved for A's separate history-remediation decision and -the P7.8 final release audit; current tracked content is clean and Git history -was not rewritten. +No open Phase-5 or P7.1 blocker remains. Public launch remains gated by pending +P7.2/P7.3 production acceptance and unstarted P7.4–P7.8. Four historical +implementation-provenance findings remain unresolved for A's separate history- +remediation decision and the P7.8 final release audit; current tracked content +is clean and Git history was not rewritten. --- Next Milestone -P7.2 — Production Migration & Deployment Path is next but unstarted. Its -current migration mechanism and privileged migration identity remain future -work. Do not start it without separate approval. +P7.2 and P7.3 require B review and one A-owned combined live validation session. +After both are accepted, P7.4 — Monitoring & Alerting is next but unstarted. --- Last Completed P7.1 — Production Security Baseline is the last operationally closed milestone. -Final decision: PASS. Phase 7 remains in progress. +P7.2/P7.3 are repository-complete only. Phase 7 remains in progress. --- @@ -880,8 +881,9 @@ P7.1 Closure State - A has formally accepted D-030. Its staged architecture and history remain unchanged. A completed the protected production execution and validation; P7.1 is closed. -- D-029 is Accepted. Phase 7 runs P7.1 through P7.8 before deferred Phase 6; - P7.2 and later work remain unstarted. +- D-029 is Accepted. Phase 7 runs P7.1 through P7.8 before deferred Phase 6. + P7.2/P7.3 now await review and live validation; P7.4 and later remain + unstarted. - Production configuration rejects committed development and test secret placeholders, short bot service credentials, pilot mode, debug/verbose log levels, and unrelated cross-boundary secret reuse without reporting values. @@ -967,11 +969,63 @@ P7.1 Closure State projection and exactly one correct Telegram notification/callback, and final health/readiness. Recent permission and secret-pattern/log-leak scans were clean. One isolated ordinary HTTP 404 was non-blocking. -- P7.2 is next but unstarted. P7.3–P7.8, Phase 6, and product expansion remain - unstarted. Backup/restore/DR remains P7.3, monitoring P7.4, DATE-001 P7.5, - network/runtime resilience P7.6, AuditLog structural immutability P7.7, and - historical provenance/final launch gate P7.8. The provenance findings remain - unresolved. +- P7.2/P7.3 repository implementation is complete and ready for B review, but + neither milestone is production-complete. D-031 provides explicit ephemeral + privileged migration plus deterministic deployment and immutable PostgreSQL/ + Redis reconciliation while retaining `wctm_runtime` for backend runtime. + D-032 provides verified custom-format backup, retention, off-host transfer, + daily scheduling templates, isolated restore, and DR procedures. A must run + the combined production-validation gates before acceptance. +- P7.4–P7.8, Phase 6, and product expansion remain unstarted. Monitoring stays + P7.4, DATE-001 P7.5, network/runtime resilience P7.6, AuditLog structural + immutability P7.7, and historical provenance/final launch gate P7.8. The + provenance findings remain unresolved. + +--- + +P7.2/P7.3 Repository Implementation State + +- P7.2 implementation commit: + `91e16fd5933ffa4e6386d649f21f9e2592833ed5 chore(ops): establish production +migration and deployment path`. +- P7.3 implementation commit: + `ddde3c705b249a29a131a3bfc96763f17b6e9cbc chore(ops): add production backup +and recovery path`. +- D-031 and D-032 are Accepted for repository implementation; D-029 and D-030 + remain Accepted and unchanged. +- `migrate` is an operations-profile Compose service built from a dedicated + backend Dockerfile target. It runs non-root/read-only, contains the exact + checked-out Prisma CLI/migrations, accepts a distinct credential only for one + wrapper-controlled invocation, rejects `wctm_runtime`, and is removed after + execution. Normal backend startup remains migration-free. +- The revision-locked deployment wrapper requires clean synchronized Git state, + Compose/config audit, a fresh verified backup, image build, explicit migration + before cutover, health/readiness, and exact restricted runtime-role proof. + Database/cache reconciliation separately preserves and verifies named volumes + while moving only PostgreSQL/Redis to the reviewed immutable references. +- Backup tooling creates atomic mode-0600 PostgreSQL custom dumps with UTC names, + SHA-256, dump readability validation, and non-secret revision/migration/ + version metadata. Explicit retention defaults to 14 valid sets, protects the + newest, and ignores unrelated/incomplete/corrupt files. The rclone hook checks + all remote sizes and requires the remote dump to match the local SHA-256 via + a compatible native hash or a locally hashed remote stream; systemd templates + schedule daily execution without committed secrets. +- Restore tooling has no production target option. It checksum-validates and + restores only into a generated PostgreSQL 16.15 container/volume with no + network, verifies exact repository migration count, public schema, and + configurable critical rows, then removes only generated resources. +- The combined A-runbook covers preflight, backup/off-host proof, migration, + PostgreSQL/Redis reconciliation, application cutover, bounded M1-M22 smoke, + post-deployment backup, isolated restore, final scans, failure handling and + DR. Initial RPO/RTO are targets of 24 hours/eight hours, not guarantees. +- Database ciphertext recovery requires the matching APP encryption key and, + only for a historical controlled rotation-window backup, any necessary + previous key. JWT/callback/internal/bot/database secrets restore their own + trust boundaries and do not decrypt stored ciphertext. +- Repository implementation is ready for B review. Neither P7.2 nor P7.3 is + production-complete until A performs and accepts live validation. No VPS, + production, secret, database, deployment, schema, migration, or product + mutation occurred during repository implementation. --- diff --git a/docs/SETUP.md b/docs/SETUP.md index b2be3da..aae0245 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -1024,3 +1024,87 @@ gh variable list --repo OWNER/REPOSITORY Do not use `gh secret set`, print environment dumps, `docker inspect` container environments, `docker compose config`, or shell tracing during this validation; those paths can expose values. + +## Production Migration, Backup, and Recovery (P7.2/P7.3) + +P7.2 and P7.3 repository implementation is documented in the executable +[P7.2/P7.3 production operations runbook](operations/P7_2_P7_3_RUNBOOK.md). +Neither milestone is production-complete until A performs and accepts the live +validation gates. + +The normal backend continues to run only as restricted `wctm_runtime` and does +not contain or invoke Prisma migration tooling at startup. Production migration +uses the separate `migrate` Compose operations profile through: + +```bash +scripts/ops/migrate-production.sh --credential-file /run/wctm/migration.env +``` + +The protected file supplies a distinct existing owner/migration URL only for +that operation. It is mode `0600`/`0400`, stays outside Git and normal runtime, +and is removed or returned to protected storage afterward. The wrapper and +container reject `wctm_runtime`; migration failure is non-zero and application +cutover must not continue. + +The supported revision-locked deployment command is: + +```bash +scripts/ops/deploy-production.sh \ + --revision FULL_REVIEWED_GIT_SHA \ + --migration-credential-file /run/wctm/migration.env \ + --backup-directory /var/backups/wctm +``` + +It requires a clean synchronized revision, config/Compose preflight, fresh +verified backup, image build, explicit migration before cutover, health/ +readiness, and restricted runtime-role verification. PostgreSQL/Redis immutable +image reconciliation is separately guarded by explicit verified-backup and +named-volume arguments. It preserves volumes and never uses `down -v`. + +Create a verified local backup with: + +```bash +scripts/ops/backup-postgres.sh --destination /var/backups/wctm +``` + +The custom-format dump is finalized atomically only after non-empty/readability +checks, receives SHA-256 and non-secret JSON metadata, and uses restrictive +permissions. Off-host copy is an optional explicit hook; same-VPS storage alone +does not meet DR: + +```bash +scripts/ops/backup-postgres.sh \ + --destination /var/backups/wctm \ + --offsite-hook scripts/ops/offsite-rclone.sh \ + --offsite-destination remote-name:wctm-production +``` + +Retention is dry-run by default and considers only complete, checksum-valid +WCTM sets. The default operational target retains 14 valid daily sets and never +deletes the newest valid set: + +```bash +scripts/ops/backup-retention.sh --directory /var/backups/wctm --keep 14 +scripts/ops/backup-retention.sh --directory /var/backups/wctm --keep 14 --apply +``` + +Daily systemd templates are under `ops/systemd/`; they are not installed or +enabled by repository implementation. A must review the service user, checkout +and backup paths, Docker access, and provider-neutral rclone destination before +following the installation commands in the operations runbook. + +Restore testing always uses a generated, non-networked PostgreSQL 16.15 +container and volume and exposes no production-target option: + +```bash +scripts/ops/restore-isolated.sh \ + --backup /var/backups/wctm/wctm-postgres-UTC-REVISION.dump \ + --critical-table tenants --critical-table stores --critical-table orders +``` + +Database recovery requires the backup, matching repository revision, and the +corresponding valid `APP_ENCRYPTION_KEY`; a previous APP key is relevant only +for a backup made during a controlled historical rotation window. JWT, +callback-signing, bot/internal, and database credentials restore their own +trust boundaries and do not decrypt stored database ciphertext. No real key or +secret belongs in the repository or operational evidence. diff --git a/docs/operations/P7_2_P7_3_RUNBOOK.md b/docs/operations/P7_2_P7_3_RUNBOOK.md new file mode 100644 index 0000000..096fa53 --- /dev/null +++ b/docs/operations/P7_2_P7_3_RUNBOOK.md @@ -0,0 +1,397 @@ +# P7.2/P7.3 Production Operations Runbook + +Status: repository implementation ready for review and A-owned production +validation. Nothing in this document marks P7.2 or P7.3 production-complete. + +## Safety and identity boundary + +- Normal backend runtime uses only `wctm_runtime`. It must remain non-owner, + non-superuser, `NOINHERIT`, without database `TEMP`, schema `CREATE`, object + ownership, or `_prisma_migrations` access. +- A distinct existing owner/migration identity is supplied only through a + protected `DATABASE_URL` file for an explicit migration operation. No + migration username or password is committed. The ephemeral migration + container is removed after execution and the credential file remains outside + the repository and normal backend environment. +- Backend startup runs only `node dist/main.js`; it never invokes Prisma + Migrate or `db push`. Prisma Migrate and the checked-out + `backend/prisma/migrations` directory remain authoritative. +- Never use `wctm_runtime` for migration. Never add migration credentials to + `.env`, the backend service, a unit file, an image, or shell history. Never + run `docker compose down -v` or delete production volumes. +- Commands below are production commands only when explicitly labelled. All + destructive recovery actions require A's separate decision. Stop on every + non-zero result unless a failure section explicitly says otherwise. + +## Operator files + +Create the migration credential file outside the checkout in a trusted shell +with tracing disabled and mode `0600` or `0400`. It contains one line only: + +```text +DATABASE_URL=postgresql://MIGRATION_IDENTITY:REDACTED@postgres:5432/DATABASE +``` + +The scripts never print this value. URL-encode password characters as required. +The entrypoint rejects a missing identity and the exact `wctm_runtime` user. +After the operation, remove the file from the deployment workspace or return it +to protected secret storage; it must not remain available to backend runtime. + +Backup sets contain: + +- `wctm-postgres--.dump` — PostgreSQL custom-format dump; +- `.dump.sha256` — SHA-256 checksum; +- `.dump.json` — non-secret timestamp, database name, repository revision, + completed migration count, size, checksum, and PostgreSQL tool/server version. + +Directories are mode `0700`; files are mode `0600`; partial files are removed +on failure and final names appear only after dump readability succeeds. + +## Supported repository commands + +Repository-side checks: + +```bash +docker compose config --quiet +bash -n scripts/ops/*.sh +``` + +Production backup (non-destructive): + +```bash +scripts/ops/backup-postgres.sh --destination /var/backups/wctm +``` + +Backup with provider-neutral off-host transfer through the supplied rclone +hook (non-destructive; rclone credentials/configuration remain outside Git): + +```bash +scripts/ops/backup-postgres.sh \ + --destination /var/backups/wctm \ + --offsite-hook scripts/ops/offsite-rclone.sh \ + --offsite-destination remote-name:wctm-production +``` + +The hook size-checks all three remote artifacts and requires the remote dump to +match the locally generated SHA-256 before success. It uses a compatible native +remote SHA-256 when rclone exposes one; otherwise it streams the remote dump +through rclone and hashes that stream locally without overwriting the local +backup. Copy, metadata, size, SHA-256, or integrity-verification failure returns +non-zero. A local backup remains valid when remote transfer fails, but the DR +gate fails. The production destination is an A-owned live-validation choice. + +Explicit migration (schema-changing, fail-fast): + +```bash +scripts/ops/migrate-production.sh \ + --credential-file /run/wctm/migration.env +``` + +Deterministic application deployment (migration and service recreation can +change production state): + +```bash +scripts/ops/deploy-production.sh \ + --revision FULL_REVIEWED_GIT_SHA \ + --migration-credential-file /run/wctm/migration.env \ + --backup-directory /var/backups/wctm +``` + +The deployment script requires the exact revision, a clean worktree, inclusion +in synchronized `origin/main`, valid Compose configuration, running services, +an all-PASS production configuration audit, and a fresh verified backup. It +builds the migration/backend/bot images, runs migration before cutover, stops +immediately on migration failure, recreates backend then bot, checks local +health/readiness, and verifies the actual backend identity and restrictions. +It does not reconcile PostgreSQL/Redis images or run product smoke by itself; +those controlled gates below surround the application deployment. + +## Controlled PostgreSQL/Redis image reconciliation + +The repository requires these exact references: + +```text +postgres:16.15-alpine3.24@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685 +redis:7.4.11-alpine3.21@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf +``` + +Before reconciliation, inspect and record the running named volumes: + +```bash +docker inspect "$(docker compose ps -q postgres)" \ + --format '{{range .Mounts}}{{.Type}} {{.Name}} {{.Destination}}{{println}}{{end}}' +docker inspect "$(docker compose ps -q redis)" \ + --format '{{range .Mounts}}{{.Type}} {{.Name}} {{.Destination}}{{println}}{{end}}' +``` + +Abort if either expected data path is not a named volume, is empty, differs +from the recorded production volume, or the fresh backup set is incomplete. +Run the guarded procedure with the exact inspected names: + +```bash +scripts/ops/reconcile-data-images.sh \ + --backup /var/backups/wctm/wctm-postgres-UTC-REVISION.dump \ + --postgres-volume EXACT_POSTGRES_VOLUME \ + --redis-volume EXACT_REDIS_VOLUME +``` + +This verifies the backup checksum, volumes, completed migration count, and +Redis key count; stops backend and bot; pulls and recreates only PostgreSQL and +Redis; preserves both named volumes; verifies PostgreSQL and Redis health, +unchanged volume attachments/data indicators, and exact image references; and +leaves application services stopped for the deployment gate. It never removes +a volume. After success, run the supported deployment command. + +On failure, do not delete or initialize either volume. Keep applications +stopped, inspect `docker compose logs postgres redis`, confirm the same volumes +are attached, and recreate the affected service with the last known-good +same-major image if safe. If database state is damaged or cannot start, use the +restore/recovery procedure; do not improvise a new production database. + +## Backup retention and schedule + +Retention is explicit and count-based. Default operational policy is 14 valid +daily sets (approximately 14 days). Dry-run first: + +```bash +scripts/ops/backup-retention.sh --directory /var/backups/wctm --keep 14 +scripts/ops/backup-retention.sh --directory /var/backups/wctm --keep 14 --apply +``` + +Only complete, checksum-valid files matching WCTM's exact timestamped naming +pattern are candidates. Unrelated files, incomplete/corrupt sets, and at least +the newest valid backup are never deleted. Nothing schedules deletion unless +the operator explicitly installs the scheduled wrapper or runs `--apply`. + +The repository templates provide a daily Ubuntu systemd timer. Review paths, +service account, Docker access, and off-host destination before installation: + +```bash +sudo install -d -o root -g wctm -m 0750 /etc/wctm +sudo install -o root -g wctm -m 0640 \ + ops/systemd/backup.conf.example /etc/wctm/backup.conf +sudo install -o root -g root -m 0644 \ + ops/systemd/wctm-backup.service /etc/systemd/system/wctm-backup.service +sudo install -o root -g root -m 0644 \ + ops/systemd/wctm-backup.timer /etc/systemd/system/wctm-backup.timer +sudo systemctl daemon-reload +sudo systemctl start wctm-backup.service +sudo systemctl status wctm-backup.service +sudo systemctl enable --now wctm-backup.timer +systemctl list-timers wctm-backup.timer +journalctl -u wctm-backup.service +``` + +These are instructions for A; C does not install or enable them. The unit +contains no secret. `/etc/wctm/backup.conf` holds paths and the optional remote +name; provider credentials use the operator's protected rclone configuration. +The service's non-zero exit and journal entry make backup/copy/retention +failures observable. Confirm the `wctm` account can access Docker without +broadening unrelated host privileges before enabling the timer. + +## Isolated restore test + +The supported restore command has no production database URL or remote target +option. It always creates a uniquely named PostgreSQL 16.15 container and +volume with `--network none`, publishes no port, verifies checksum and dump +readability, restores with `--no-owner --no-privileges`, compares completed +migrations to the checked-out repository, checks public tables, and reports +configured critical table row counts. It removes only its generated isolated +container/volume. The optional diagnostic retention flag never targets an +existing resource. + +```bash +scripts/ops/restore-isolated.sh \ + --backup /var/backups/wctm/wctm-postgres-UTC-REVISION.dump \ + --critical-table tenants \ + --critical-table users \ + --critical-table memberships \ + --critical-table stores \ + --critical-table orders \ + --critical-table audit_logs +``` + +Compare the reported counts to a source-side, non-secret count record captured +at backup time when recovery validation requires exact continuity. Application- +compatible validation may use a separately started test backend attached only +to this isolated database with synthetic credentials and no external Telegram +or WooCommerce calls. It is optional and must never reuse production service +secrets or expose the restore target. + +For diagnosis only, `--keep-on-failure` retains the uniquely generated isolated +resources and prints their names. Remove only those exact names after review. + +## Recovery dependencies + +A database dump alone cannot recover encrypted Store credentials, webhook +secrets, pending Telegram note text, or saved search text. Recovery requires: + +- the valid `APP_ENCRYPTION_KEY` corresponding to the backup; +- only during a historically applicable controlled rotation window, the + required decrypt-only previous APP key; +- the database backup, checksum, and metadata; +- the matching repository/application revision and migration history; +- securely restored JWT, backend-bot, callback-signing, bot, runtime database, + and connector-related secrets needed for their own authentication functions. + +JWT, callback-signing, bot/internal API, and database credentials do not decrypt +stored application ciphertext. They restore session/signing/service/database +trust boundaries. Never print, encode into docs, or commit any recovery secret. + +## Disaster-recovery sequences + +### Application/container loss with VPS and volumes intact + +1. Preserve and inspect named volumes; do not initialize or delete them. +2. Check out the exact reviewed revision and restore protected runtime config. +3. Validate Compose, exact images, disk, PostgreSQL/Redis health, and backup. +4. Run explicit migrations only if the repository revision requires them. +5. Recreate backend and bot, then pass config audit, runtime-role verification, + health/readiness, and bounded smoke. + +### Failed or corrupted deployment + +1. Stop cutover on build/migration/readiness failure and preserve logs. +2. Before migration, continue running the old application; no DB rollback is + required. After migration, an old image may be incompatible with the new + schema, so do not blindly redeploy it. +3. If the migration is additive and verified compatible, A may restore the + prior reviewed application image. Otherwise select database recovery below. +4. Prisma migrations are not universally reversible. Never synthesize a down + migration during an incident. + +### Database loss or corruption + +1. Stop backend and bot to prevent writes. Preserve failed database data and + logs for diagnosis. +2. Select the newest verified backup compatible with the protected APP key and + repository revision; first pass the isolated restore procedure. +3. A must explicitly authorize creation/replacement of the production target. + The repository provides no production-overwrite command by default. +4. Restore custom dump, validate migrations/schema/critical counts, restore the + restricted runtime grants, and verify `wctm_runtime` before starting apps. +5. Deploy the matching application revision, then run full recovery smoke. + +### Complete VPS loss + +1. Provision supported Ubuntu/Docker/Compose/Caddy capacity with private + PostgreSQL/Redis/bot networking, loopback backend publication, TLS, firewall, + time synchronization, disk capacity, and a non-root operator account. +2. Check out the reviewed revision. Restore protected configuration and keys + from secret storage, never from the database backup. +3. Initialize exact immutable PostgreSQL/Redis images and new named volumes. +4. Restore the verified off-host database backup, validate schema/migrations and + critical counts, then recreate the restricted runtime role/grants. +5. Run explicit migrations only if intentionally advancing beyond the restored + revision. Deploy backend and Telegram bot and restore Caddy routing. +6. Validate Store/connector state. Existing WooCommerce hooks and Store secrets + normally survive through the database; reconcile only through existing M7/M8 + paths if evidence shows otherwise. Restore the Telegram bot token and shared + backend-bot key coherently; links persist in the restored database. +7. Pass config audit, role verification, health/readiness, bounded product + smoke, permission/error/secret scans, and a new post-recovery backup. + +Redis queue/transient state may not represent every in-flight effect after a +disaster. Preserve existing at-least-once/idempotency rules and inspect durable +database delivery/event state before replaying anything. Do not blindly resend +Telegram or WooCommerce effects. + +## Initial recovery objectives + +These are operational targets, not contractual guarantees or an SLA: + +- RPO target: at most 24 hours with the daily verified off-host backup; reduce + the timer interval only after capacity and restore testing support it. +- RTO target: eight hours for a single-host recovery with infrastructure, + secrets, reviewed images, and a verified backup available. +- Retention target: 14 valid daily local sets plus an independently retained + off-host copy policy chosen by A/provider. Same-VPS copies alone do not meet + the DR target. + +## A-owned combined production validation session + +Every gate is fail-fast. A may record a single final evidence bundle instead of +reporting after each successful command. Stop immediately for a failed backup, +off-host copy, migration, identity check, data continuity check, health or +readiness check, secret exposure, or unexplained permission/migration error. + +### Gate 1 — preflight + +1. Synchronize `main`, verify the reviewed full SHA, clean worktree, and its + containment in `origin/main`; run `docker compose config --quiet`. +2. Record `docker compose ps`, current exact image strings, disk/inode capacity, + PostgreSQL/Redis named volumes, and current completed migration count. +3. Pass current health/readiness and `security:config-audit`. +4. Pass `scripts/ops/verify-runtime-role.sh`; runtime must be `wctm_runtime`. + +### Gate 2 — fresh backup + +1. Run `backup-postgres.sh` to the protected backup directory. +2. Confirm non-empty dump, `.sha256`, `.json`, mode, checksum, migration count, + and `pg_restore --list` success as reported by the command. +3. Run the off-site hook and require its verified PASS. Stop if remote transfer + fails; do not call the gate successful because the local copy exists. + +### Gate 3 — migration path + +1. Place the distinct owner/migration URL in the protected one-line file. +2. Run `migrate-production.sh`; require Prisma Migrate success/current status. +3. Confirm the ephemeral migration container is absent afterward and remove or + secure the credential file outside normal runtime. +4. Re-run runtime-role verification; backend remains `wctm_runtime` with no + migration-table access. A failed migration stops before application cutover. + +### Gate 4 — immutable PostgreSQL/Redis reconciliation + +1. Reconfirm Gate 2 backup and exact volume names. +2. Run `reconcile-data-images.sh` with those explicit values. +3. Require exact reviewed image references, same named volumes, unchanged + migration count and Redis key count, PostgreSQL readiness, and Redis PONG. +4. Applications intentionally remain stopped. On any failure, preserve volumes + and enter the documented abort/recovery path. + +### Gate 5 — application deployment + +1. Run `deploy-production.sh` at the reviewed SHA. Its second fresh backup and + repeat migration are intentional and must be safe/idempotent. +2. Require clean backend/bot startup, all-PASS config audit, local health and + readiness, restricted-role PASS, current APP key only, and no migration + credential in backend runtime. + +### Gate 6 — bounded functional smoke + +Validate only: web login; authenticated Tenant read; Telegram `/status`; +`/orders` plus detail; `/search`; `/report`; `/stock`; and Store/connector +health. A new synthetic customer/order mutation is not required unless another +gate provides insufficient evidence. + +### Gate 7 — post-deployment backup + +Create a fresh local custom-format backup and verified off-host copy from the +new state. Record filename, UTC timestamp, revision, size, migration count, and +checksum only. + +### Gate 8 — isolated restore + +Run `restore-isolated.sh` against the Gate 7 backup with the critical tables +listed above. Require checksum/readability, isolated no-network PostgreSQL, +restore success, exact repository migration count, expected schema/tables, +credible critical row counts, and clean generated-resource teardown. + +### Gate 9 — final scans + +Pass final health/readiness, config audit, runtime-role check, and bounded recent +logs for database permission, migration, backup/restore, startup, and secret- +pattern failures. Do not paste matching secret-bearing output. Record that no +production volume was removed and that P7.2/P7.3 acceptance remains A-owned. + +## Failure matrix + +| Failure | Required response | +| ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| Image build/pull | No migration/cutover if build fails; keep current services. A failed data-image pull stops before recreation. | +| Migration before cutover | Stop. Keep current app if still compatible/running. Remove ephemeral migration container. Diagnose with secret-safe logs. | +| Backend startup/readiness | Keep bot stopped or restore coherent backend/bot application images only when schema-compatible. Do not roll back schema automatically. | +| PostgreSQL/Redis recreation | Keep apps stopped, preserve named volumes, inspect logs/mounts, retry known-good same-major image or enter restore. | +| Migration/data damage | Stop all writers and use the verified P7.3 recovery process. Prisma rollback is not assumed. | +| Backup/off-host/restore test | Deployment/acceptance stops; preserve the last valid backup and diagnose. Never report partial success as DR readiness. | diff --git a/ops/systemd/backup.conf.example b/ops/systemd/backup.conf.example new file mode 100644 index 0000000..eabc379 --- /dev/null +++ b/ops/systemd/backup.conf.example @@ -0,0 +1,5 @@ +# Install as /etc/wctm/backup.conf with root:wctm ownership and mode 0640. +WCTM_BACKUP_DIRECTORY=/var/backups/wctm +WCTM_BACKUP_RETENTION_COUNT=14 +# Optional provider-neutral rclone remote. Configure credentials outside Git. +# WCTM_OFFSITE_DESTINATION=remote-name:wctm-production diff --git a/ops/systemd/wctm-backup.service b/ops/systemd/wctm-backup.service new file mode 100644 index 0000000..dfe3b8b --- /dev/null +++ b/ops/systemd/wctm-backup.service @@ -0,0 +1,21 @@ +[Unit] +Description=WCTM PostgreSQL backup and verified off-host copy +Requires=docker.service +After=docker.service + +[Service] +Type=oneshot +User=wctm +Group=wctm +WorkingDirectory=/srv/wctm +EnvironmentFile=/etc/wctm/backup.conf +ExecStart=/srv/wctm/scripts/ops/scheduled-backup.sh +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/backups/wctm + +[Install] +WantedBy=multi-user.target diff --git a/ops/systemd/wctm-backup.timer b/ops/systemd/wctm-backup.timer new file mode 100644 index 0000000..c62b62a --- /dev/null +++ b/ops/systemd/wctm-backup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run the WCTM PostgreSQL backup daily + +[Timer] +OnCalendar=*-*-* 02:15:00 UTC +Persistent=true +RandomizedDelaySec=15m +Unit=wctm-backup.service + +[Install] +WantedBy=timers.target diff --git a/package.json b/package.json index 2d06173..bf536c2 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "entitlement:manage": "npm run entitlement:manage --workspace backend --", "security:config-audit": "npm run security:config-audit --workspace backend", "security:rotate-encryption-key": "npm run security:rotate-encryption-key --workspace backend --", + "test:operations": "scripts/ops/test-operations.sh", "format": "prettier --write \".eslintrc.js\" \"*.json\" \"*.md\" \"*.yml\" \".github/workflows/*.yml\" \"backend/**/*.{json,ts}\" \"backend/test/**/*.cjs\" \"telegram-bot/**/*.{json,ts}\" \"docs/SETUP.md\"", "format:check": "prettier --check \".eslintrc.js\" \"*.json\" \"*.md\" \"*.yml\" \".github/workflows/*.yml\" \"backend/**/*.{json,ts}\" \"backend/test/**/*.cjs\" \"telegram-bot/**/*.{json,ts}\" \"docs/SETUP.md\"", "test": "npm run test --workspaces --if-present", diff --git a/scripts/ops/backup-postgres.sh b/scripts/ops/backup-postgres.sh new file mode 100755 index 0000000..b40d177 --- /dev/null +++ b/scripts/ops/backup-postgres.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +usage() { + echo >&2 'usage: backup-postgres.sh --destination DIRECTORY [--offsite-hook EXECUTABLE --offsite-destination DESTINATION]' + exit 64 +} + +destination='' +offsite_hook='' +offsite_destination='' +while (($#)); do + case "$1" in + --destination) + (($# >= 2)) || usage + destination=$2 + shift 2 + ;; + --offsite-hook) + (($# >= 2)) || usage + offsite_hook=$2 + shift 2 + ;; + --offsite-destination) + (($# >= 2)) || usage + offsite_destination=$2 + shift 2 + ;; + *) usage ;; + esac +done + +[[ -n "$destination" && "$destination" != '/' ]] || { + echo >&2 'backup refused: destination must be an explicit non-root directory' + exit 64 +} +if [[ -n "$offsite_hook" || -n "$offsite_destination" ]]; then + [[ -n "$offsite_hook" && -x "$offsite_hook" && -n "$offsite_destination" ]] || { + echo >&2 'backup refused: off-site hook and destination must be supplied together' + exit 64 + } +fi + +mkdir -p -- "$destination" +chmod 0700 "$destination" +destination=$(cd "$destination" && pwd -P) + +container_id=${WCTM_POSTGRES_CONTAINER:-$(docker compose ps -q postgres)} +[[ -n "$container_id" ]] || { + echo >&2 'backup failed: PostgreSQL container is not running' + exit 1 +} + +database_name=$(docker exec "$container_id" sh -c 'exec psql -XAtq -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select current_database()"') +[[ "$database_name" =~ ^[A-Za-z0-9_.-]+$ ]] || { + echo >&2 'backup failed: database name is not safe for metadata' + exit 1 +} + +created_at=$(date -u '+%Y-%m-%dT%H:%M:%SZ') +timestamp=$(date -u '+%Y%m%dT%H%M%SZ') +revision=$(git rev-parse HEAD 2>/dev/null || printf 'unknown') +revision_short=${revision:0:12} +base="wctm-postgres-${timestamp}-${revision_short}" +dump_path="$destination/$base.dump" +checksum_path="$dump_path.sha256" +metadata_path="$dump_path.json" + +[[ ! -e "$dump_path" && ! -e "$checksum_path" && ! -e "$metadata_path" ]] || { + echo >&2 'backup failed: timestamped destination already exists' + exit 1 +} + +partial_dump=$(mktemp "$destination/.${base}.dump.partial.XXXXXX") +partial_checksum=$(mktemp "$destination/.${base}.sha256.partial.XXXXXX") +partial_metadata=$(mktemp "$destination/.${base}.json.partial.XXXXXX") +cleanup() { + rm -f -- "$partial_dump" "$partial_checksum" "$partial_metadata" +} +trap cleanup EXIT HUP INT TERM + +docker exec "$container_id" sh -c 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=custom --no-owner --no-privileges' >"$partial_dump" +[[ -s "$partial_dump" ]] || { + echo >&2 'backup failed: pg_dump produced an empty file' + exit 1 +} +docker exec -i "$container_id" pg_restore --list <"$partial_dump" >/dev/null + +checksum=$(sha256sum "$partial_dump" | awk '{print $1}') +[[ "$checksum" =~ ^[0-9a-f]{64}$ ]] || { + echo >&2 'backup failed: invalid SHA-256 result' + exit 1 +} +size_bytes=$(wc -c <"$partial_dump" | tr -d ' ') +migration_count=$(docker exec "$container_id" sh -c 'exec psql -XAtq -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select count(*) from public._prisma_migrations where finished_at is not null and rolled_back_at is null"') +[[ "$migration_count" =~ ^[0-9]+$ ]] || { + echo >&2 'backup failed: migration count is unavailable' + exit 1 +} +server_version=$(docker exec "$container_id" sh -c 'exec psql -XAtq -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "show server_version"') +tool_version=$(docker exec "$container_id" pg_dump --version | tr -d '\r\n') + +printf '%s %s\n' "$checksum" "$(basename "$dump_path")" >"$partial_checksum" +cat >"$partial_metadata" <"$WCTM_BACKUP_RESULT_FILE" + chmod 0600 "$WCTM_BACKUP_RESULT_FILE" +fi + +echo "backup: PASS file=$(basename "$dump_path") size_bytes=$size_bytes migrations=$migration_count checksum=$checksum" diff --git a/scripts/ops/backup-retention.sh b/scripts/ops/backup-retention.sh new file mode 100755 index 0000000..ba6f649 --- /dev/null +++ b/scripts/ops/backup-retention.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo >&2 'usage: backup-retention.sh --directory DIRECTORY [--keep 14] [--apply]' + exit 64 +} + +directory='' +keep=14 +apply=false +while (($#)); do + case "$1" in + --directory) (($# >= 2)) || usage; directory=$2; shift 2 ;; + --keep) (($# >= 2)) || usage; keep=$2; shift 2 ;; + --apply) apply=true; shift ;; + *) usage ;; + esac +done + +[[ -d "$directory" && "$directory" != '/' ]] || { + echo >&2 'retention refused: directory must be an existing non-root directory' + exit 64 +} +[[ "$keep" =~ ^[0-9]+$ ]] && ((keep >= 1)) || { + echo >&2 'retention refused: --keep must be at least 1' + exit 64 +} + +directory=$(cd "$directory" && pwd -P) +valid=() +while IFS= read -r dump; do + [[ -f "$dump.sha256" && -f "$dump.json" ]] || continue + if (cd "$directory" && sha256sum --check --status "$(basename "$dump.sha256")"); then + valid+=("$dump") + fi +done < <(find "$directory" -maxdepth 1 -type f -name 'wctm-postgres-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]Z-*.dump' -print | LC_ALL=C sort -r) + +deleted=0 +for ((index=keep; index<${#valid[@]}; index++)); do + dump=${valid[$index]} + if $apply; then + rm -- "$dump" "$dump.sha256" "$dump.json" + else + echo "would remove backup set: $(basename "$dump")" + fi + deleted=$((deleted + 1)) +done + +mode=dry-run +$apply && mode=applied +echo "retention: PASS mode=$mode valid_sets=${#valid[@]} kept_at_least=$keep selected_for_removal=$deleted" diff --git a/scripts/ops/deploy-production.sh b/scripts/ops/deploy-production.sh new file mode 100755 index 0000000..509d264 --- /dev/null +++ b/scripts/ops/deploy-production.sh @@ -0,0 +1,66 @@ +#!/bin/sh +set -eu +umask 077 + +usage() { + echo >&2 'usage: deploy-production.sh --revision GIT_SHA --migration-credential-file PATH --backup-directory PATH' + exit 64 +} + +[ "$#" -eq 6 ] || usage +[ "$1" = '--revision' ] || usage +expected_revision=$2 +[ "$3" = '--migration-credential-file' ] || usage +migration_credential_file=$4 +[ "$5" = '--backup-directory' ] || usage +backup_directory=$6 + +case "$expected_revision" in + *[!0-9a-fA-F]*|'') + echo >&2 'deployment refused: revision must be a hexadecimal Git object ID' + exit 64 + ;; +esac + +actual_revision=$(git rev-parse HEAD) +[ "$actual_revision" = "$expected_revision" ] || { + echo >&2 'deployment refused: checked-out revision does not match --revision' + exit 1 +} + +[ -z "$(git status --porcelain)" ] || { + echo >&2 'deployment refused: repository worktree is not clean' + exit 1 +} + +git merge-base --is-ancestor "$expected_revision" origin/main || { + echo >&2 'deployment refused: revision is not contained in synchronized origin/main' + exit 1 +} + +docker compose config --quiet +[ -n "$(docker compose ps --status running -q postgres)" ] && [ -n "$(docker compose ps --status running -q redis)" ] || { + echo >&2 'deployment refused: PostgreSQL and Redis must be running' + exit 1 +} +if [ -n "$(docker compose ps -q backend)" ]; then + docker compose exec -T backend npm run security:config-audit +else + echo 'pre-cutover config audit: previously required by Gate 1 (backend currently stopped)' +fi + +scripts/ops/backup-postgres.sh --destination "$backup_directory" + +docker compose build migrate backend telegram-bot + +scripts/ops/migrate-production.sh --credential-file "$migration_credential_file" + +docker compose up -d --no-deps --force-recreate backend +docker compose up -d --no-deps --force-recreate telegram-bot + +runtime_port=${PORT:-3000} +curl --fail --silent --show-error "http://127.0.0.1:$runtime_port/api/health" >/dev/null +curl --fail --silent --show-error "http://127.0.0.1:$runtime_port/api/health/readiness" >/dev/null +scripts/ops/verify-runtime-role.sh + +echo "deployment path: PASS revision=$actual_revision" diff --git a/scripts/ops/migrate-production.sh b/scripts/ops/migrate-production.sh new file mode 100755 index 0000000..96619dc --- /dev/null +++ b/scripts/ops/migrate-production.sh @@ -0,0 +1,60 @@ +#!/bin/sh +set -eu +umask 077 + +usage() { + echo >&2 'usage: migrate-production.sh --credential-file PATH' + exit 64 +} + +[ "$#" -eq 2 ] || usage +[ "$1" = '--credential-file' ] || usage +credential_file=$2 + +[ -f "$credential_file" ] || { + echo >&2 'migration refused: credential file does not exist' + exit 66 +} + +file_mode=$(stat -c '%a' "$credential_file" 2>/dev/null || stat -f '%Lp' "$credential_file") +case "$file_mode" in + 600|400) ;; + *) + echo >&2 'migration refused: credential file mode must be 0600 or 0400' + exit 65 + ;; +esac + +database_url='' +line_count=0 +while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|'#'*) continue ;; + DATABASE_URL=*) + line_count=$((line_count + 1)) + database_url=${line#DATABASE_URL=} + ;; + *) + echo >&2 'migration refused: credential file may contain only DATABASE_URL and comments' + exit 65 + ;; + esac +done < "$credential_file" + +[ "$line_count" -eq 1 ] && [ -n "$database_url" ] || { + echo >&2 'migration refused: credential file must contain exactly one non-empty DATABASE_URL' + exit 65 +} + +cleanup() { + database_url='' + unset database_url + docker compose --profile operations rm --force --stop migrate >/dev/null 2>&1 || true +} +trap cleanup EXIT HUP INT TERM + +echo 'starting explicit Prisma migration operation' +DATABASE_URL="$database_url" WCTM_MIGRATION_OPERATION=P7_2_EXPLICIT_MIGRATION \ + docker compose --profile operations run --rm --no-deps \ + -e DATABASE_URL -e WCTM_MIGRATION_OPERATION migrate +echo 'migration operation: PASS' diff --git a/scripts/ops/offsite-rclone.sh b/scripts/ops/offsite-rclone.sh new file mode 100755 index 0000000..60525e8 --- /dev/null +++ b/scripts/ops/offsite-rclone.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# != 4)); then + echo >&2 'usage: offsite-rclone.sh DUMP CHECKSUM METADATA RCLONE_DESTINATION' + exit 64 +fi + +dump=$1 +checksum=$2 +metadata=$3 +destination=${4%/} + +for file in "$dump" "$checksum" "$metadata"; do + [[ -f "$file" ]] || { + echo >&2 'off-site copy failed: backup set is incomplete' + exit 66 + } +done +command -v rclone >/dev/null 2>&1 || { + echo >&2 'off-site copy failed: rclone is unavailable' + exit 69 +} + +read -r expected_dump_sha256 expected_dump_name checksum_extra <"$checksum" +[[ "$expected_dump_sha256" =~ ^[0-9a-fA-F]{64}$ && + "$expected_dump_name" = "$(basename "$dump")" && + -z "${checksum_extra:-}" ]] || { + echo >&2 'off-site copy failed: local checksum record is invalid' + exit 65 +} +expected_dump_sha256=$(printf '%s' "$expected_dump_sha256" | tr 'A-F' 'a-f') +local_dump_sha256=$(sha256sum "$dump" | awk '{print $1}') +[[ "$local_dump_sha256" = "$expected_dump_sha256" ]] || { + echo >&2 'off-site copy failed: local dump checksum verification failed' + exit 1 +} + +for file in "$dump" "$checksum" "$metadata"; do + name=$(basename "$file") + rclone copyto -- "$file" "$destination/$name" + local_size=$(wc -c <"$file" | tr -d ' ') + remote_size=$(rclone size --json "$destination/$name" | sed -n 's/.*"bytes":\([0-9][0-9]*\).*/\1/p') + [[ "$remote_size" =~ ^[0-9]+$ && "$remote_size" = "$local_size" ]] || { + echo >&2 "off-site copy failed: size verification failed for $name" + exit 1 + } +done + +remote_dump="$destination/$(basename "$dump")" +remote_dump_sha256='' +verification_method='' +if native_hash_output=$(rclone hashsum sha256 "$remote_dump" 2>/dev/null); then + native_hash=$(printf '%s\n' "$native_hash_output" | awk 'NR == 1 { print tolower($1) }') + if [[ "$native_hash" =~ ^[0-9a-f]{64}$ ]]; then + remote_dump_sha256=$native_hash + verification_method='native-sha256' + fi +fi + +if [[ -z "$remote_dump_sha256" ]]; then + if ! remote_dump_sha256=$(rclone cat "$remote_dump" | sha256sum | awk '{print $1}'); then + echo >&2 'off-site copy failed: remote content verification is unavailable' + exit 1 + fi + [[ "$remote_dump_sha256" =~ ^[0-9a-f]{64}$ ]] || { + echo >&2 'off-site copy failed: remote content verification is invalid' + exit 1 + } + verification_method='streamed-sha256' +fi + +[[ "$remote_dump_sha256" = "$expected_dump_sha256" ]] || { + echo >&2 'off-site copy failed: remote dump SHA-256 mismatch' + exit 1 +} + +echo "off-site copy: PASS files=3 content_sha256=verified method=$verification_method destination-configured=true" diff --git a/scripts/ops/reconcile-data-images.sh b/scripts/ops/reconcile-data-images.sh new file mode 100755 index 0000000..055d75b --- /dev/null +++ b/scripts/ops/reconcile-data-images.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo >&2 'usage: reconcile-data-images.sh --backup FILE --postgres-volume NAME --redis-volume NAME' + exit 64 +} + +backup='' +expected_postgres_volume='' +expected_redis_volume='' +while (($#)); do + case "$1" in + --backup) (($# >= 2)) || usage; backup=$2; shift 2 ;; + --postgres-volume) (($# >= 2)) || usage; expected_postgres_volume=$2; shift 2 ;; + --redis-volume) (($# >= 2)) || usage; expected_redis_volume=$2; shift 2 ;; + *) usage ;; + esac +done + +[[ -f "$backup" && -f "$backup.sha256" && -f "$backup.json" ]] || { + echo >&2 'reconciliation refused: complete verified backup set is required' + exit 65 +} +(cd "$(dirname "$backup")" && sha256sum --check --status "$(basename "$backup.sha256")") || { + echo >&2 'reconciliation refused: backup checksum failed' + exit 1 +} +for volume in "$expected_postgres_volume" "$expected_redis_volume"; do + [[ "$volume" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]+$ ]] || { + echo >&2 'reconciliation refused: explicit named volumes are required' + exit 64 + } +done + +postgres_container=$(docker compose ps -q postgres) +redis_container=$(docker compose ps -q redis) +[[ -n "$postgres_container" && -n "$redis_container" ]] || { + echo >&2 'reconciliation refused: PostgreSQL and Redis must be running for preflight' + exit 1 +} + +postgres_volume=$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/postgresql/data"}}{{.Name}}{{end}}{{end}}' "$postgres_container") +redis_volume=$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}' "$redis_container") +[[ "$postgres_volume" = "$expected_postgres_volume" && "$redis_volume" = "$expected_redis_volume" ]] || { + echo >&2 'reconciliation refused: running named volumes do not match explicit expected values' + exit 1 +} + +before_migrations=$(docker exec "$postgres_container" sh -c 'exec psql -XAtq -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select count(*) from public._prisma_migrations where finished_at is not null and rolled_back_at is null"') +before_redis_keys=$(docker exec "$redis_container" redis-cli --raw DBSIZE) + +failure() { + echo >&2 'data-image reconciliation stopped: keep both named volumes; inspect service logs and use the documented recovery path' +} +trap failure ERR + +docker compose stop backend telegram-bot +docker compose pull postgres redis +docker compose up -d --no-deps --force-recreate postgres redis + +for _ in $(seq 1 60); do + docker compose exec -T postgres sh -c 'pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"' >/dev/null 2>&1 && break + sleep 1 +done +docker compose exec -T postgres sh -c 'pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"' >/dev/null +[[ $(docker compose exec -T redis redis-cli --raw PING | tr -d '\r') = PONG ]] + +postgres_container=$(docker compose ps -q postgres) +redis_container=$(docker compose ps -q redis) +[[ $(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/postgresql/data"}}{{.Name}}{{end}}{{end}}' "$postgres_container") = "$expected_postgres_volume" ]] +[[ $(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}' "$redis_container") = "$expected_redis_volume" ]] + +after_migrations=$(docker exec "$postgres_container" sh -c 'exec psql -XAtq -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "select count(*) from public._prisma_migrations where finished_at is not null and rolled_back_at is null"') +after_redis_keys=$(docker exec "$redis_container" redis-cli --raw DBSIZE) +[[ "$after_migrations" = "$before_migrations" && "$after_redis_keys" = "$before_redis_keys" ]] + +postgres_image=$(docker inspect --format '{{.Config.Image}}' "$postgres_container") +redis_image=$(docker inspect --format '{{.Config.Image}}' "$redis_container") +[[ "$postgres_image" = 'postgres:16.15-alpine3.24@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685' ]] +[[ "$redis_image" = 'redis:7.4.11-alpine3.21@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf' ]] + +trap - ERR +echo "data-image reconciliation: PASS migrations=$after_migrations redis_keys=$after_redis_keys volumes_preserved=true applications_stopped=true" diff --git a/scripts/ops/restore-isolated.sh b/scripts/ops/restore-isolated.sh new file mode 100755 index 0000000..810ba26 --- /dev/null +++ b/scripts/ops/restore-isolated.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +usage() { + echo >&2 'usage: restore-isolated.sh --backup FILE [--critical-table TABLE ...] [--keep-on-failure]' + exit 64 +} + +backup='' +keep_on_failure=false +critical_tables=() +while (($#)); do + case "$1" in + --backup) (($# >= 2)) || usage; backup=$2; shift 2 ;; + --critical-table) (($# >= 2)) || usage; critical_tables+=("$2"); shift 2 ;; + --keep-on-failure) keep_on_failure=true; shift ;; + *) usage ;; + esac +done + +[[ -f "$backup" && -s "$backup" ]] || { + echo >&2 'restore refused: explicit non-empty backup file is required' + exit 64 +} +[[ -f "$backup.sha256" && -f "$backup.json" ]] || { + echo >&2 'restore refused: checksum and metadata sidecars are required' + exit 65 +} +for table in "${critical_tables[@]}"; do + [[ "$table" =~ ^[a-z_][a-z0-9_]*$ ]] || { + echo >&2 'restore refused: critical table names must be lowercase SQL identifiers' + exit 64 + } +done + +backup=$(cd "$(dirname "$backup")" && pwd -P)/$(basename "$backup") +(cd "$(dirname "$backup")" && sha256sum --check --status "$(basename "$backup.sha256")") || { + echo >&2 'restore refused: checksum verification failed' + exit 1 +} + +image='postgres:16.15-alpine3.24@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685' +suffix="$(date -u '+%Y%m%dT%H%M%SZ')-$$" +container="wctm-restore-test-$suffix" +volume="wctm_restore_test_$suffix" +database='wctm_isolated_restore' +password="restore-test-$suffix" +completed=false + +cleanup() { + status=$? + if $completed || ! $keep_on_failure; then + docker rm --force "$container" >/dev/null 2>&1 || true + case "$volume" in + wctm_restore_test_*) docker volume rm "$volume" >/dev/null 2>&1 || true ;; + esac + else + echo >&2 "isolated restore resources retained for diagnosis: container=$container volume=$volume" + fi + exit "$status" +} +trap cleanup EXIT HUP INT TERM + +docker volume create "$volume" >/dev/null +docker run -d --name "$container" --network none \ + --mount "type=volume,source=$volume,destination=/var/lib/postgresql/data" \ + -e POSTGRES_DB="$database" -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD="$password" \ + "$image" >/dev/null + +stable_ready=0 +for _ in $(seq 1 60); do + if docker exec "$container" pg_isready -U postgres -d "$database" >/dev/null 2>&1; then + stable_ready=$((stable_ready + 1)) + ((stable_ready >= 3)) && break + else + stable_ready=0 + fi + sleep 1 +done +((stable_ready >= 3)) || { + echo >&2 'restore failed: isolated PostgreSQL did not become stably ready' + exit 1 +} +docker exec -i "$container" pg_restore --list <"$backup" >/dev/null +docker exec -i "$container" pg_restore -U postgres -d "$database" --exit-on-error --no-owner --no-privileges <"$backup" + +expected_migrations=$(find backend/prisma/migrations -mindepth 2 -maxdepth 2 -name migration.sql | wc -l | tr -d ' ') +restored_migrations=$(docker exec "$container" psql -XAtq -v ON_ERROR_STOP=1 -U postgres -d "$database" -c 'select count(*) from public._prisma_migrations where finished_at is not null and rolled_back_at is null') +[[ "$restored_migrations" = "$expected_migrations" ]] || { + echo >&2 'restore verification failed: migration count differs from repository state' + exit 1 +} + +table_count=$(docker exec "$container" psql -XAtq -v ON_ERROR_STOP=1 -U postgres -d "$database" -c "select count(*) from information_schema.tables where table_schema='public' and table_type='BASE TABLE'") +[[ "$table_count" =~ ^[1-9][0-9]*$ ]] || { + echo >&2 'restore verification failed: public schema contains no base tables' + exit 1 +} + +for table in "${critical_tables[@]}"; do + exists=$(docker exec "$container" psql -XAtq -v ON_ERROR_STOP=1 -U postgres -d "$database" -c "select count(*) from information_schema.tables where table_schema='public' and table_name='$table'") + [[ "$exists" = '1' ]] || { + echo >&2 "restore verification failed: critical table is absent: $table" + exit 1 + } + count=$(docker exec "$container" psql -XAtq -v ON_ERROR_STOP=1 -U postgres -d "$database" -c "select count(*) from public.\"$table\"") + echo "critical table: $table rows=$count" +done + +completed=true +echo "isolated restore: PASS migrations=$restored_migrations public_tables=$table_count network=none" diff --git a/scripts/ops/run-migrations.sh b/scripts/ops/run-migrations.sh new file mode 100755 index 0000000..66aaa78 --- /dev/null +++ b/scripts/ops/run-migrations.sh @@ -0,0 +1,32 @@ +#!/bin/sh +set -eu + +if [ "${WCTM_MIGRATION_OPERATION:-}" != "P7_2_EXPLICIT_MIGRATION" ]; then + echo >&2 'migration refused: use scripts/ops/migrate-production.sh' + exit 64 +fi + +if [ -z "${DATABASE_URL:-}" ]; then + echo >&2 'migration refused: DATABASE_URL is unavailable' + exit 64 +fi + +node <<'NODE' +const value = process.env.DATABASE_URL; +try { + const url = new URL(value); + if (!['postgres:', 'postgresql:'].includes(url.protocol)) { + throw new Error('unsupported scheme'); + } + const username = decodeURIComponent(url.username); + if (!username || username === 'wctm_runtime' || !url.password) { + throw new Error('runtime or missing identity'); + } +} catch { + console.error('migration refused: DATABASE_URL must use a distinct PostgreSQL migration identity'); + process.exit(64); +} +NODE + +echo 'migration identity boundary: PASS (distinct non-runtime identity supplied for this operation)' +exec npm run prisma:migrate:deploy --workspace=@wc-telegram/backend diff --git a/scripts/ops/scheduled-backup.sh b/scripts/ops/scheduled-backup.sh new file mode 100755 index 0000000..98139d0 --- /dev/null +++ b/scripts/ops/scheduled-backup.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +: "${WCTM_BACKUP_DIRECTORY:?WCTM_BACKUP_DIRECTORY is required}" +retention_count=${WCTM_BACKUP_RETENTION_COUNT:-14} +result_file=$(mktemp) +trap 'rm -f -- "$result_file"' EXIT HUP INT TERM + +WCTM_BACKUP_RESULT_FILE=$result_file scripts/ops/backup-postgres.sh --destination "$WCTM_BACKUP_DIRECTORY" +backup=$(cat "$result_file") + +if [[ -n "${WCTM_OFFSITE_DESTINATION:-}" ]]; then + scripts/ops/offsite-rclone.sh "$backup" "$backup.sha256" "$backup.json" "$WCTM_OFFSITE_DESTINATION" +fi + +scripts/ops/backup-retention.sh --directory "$WCTM_BACKUP_DIRECTORY" --keep "$retention_count" --apply +echo 'scheduled backup: PASS' diff --git a/scripts/ops/test-fixtures/rclone b/scripts/ops/test-fixtures/rclone new file mode 100755 index 0000000..9efcd98 --- /dev/null +++ b/scripts/ops/test-fixtures/rclone @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +command_name=${1:-} +shift || true + +remote_path() { + case "$1" in + testremote:*) printf '%s\n' "${1#testremote:}" ;; + *) echo >&2 'test rclone refused a non-test remote'; exit 64 ;; + esac +} + +case "$command_name" in + copyto) + [[ ${1:-} = '--' ]] && shift + source_path=${1:?} + target_path=$(remote_path "${2:?}") + mkdir -p -- "$(dirname "$target_path")" + cp -- "$source_path" "$target_path" + if [[ ${WCTM_TEST_RCLONE_CORRUPT_DUMP:-false} = true && "$target_path" = *.dump ]]; then + first_byte=$(od -An -tu1 -N1 "$target_path" | tr -d ' ') + replacement='X' + [[ "$first_byte" = 88 ]] && replacement='Y' + printf '%s' "$replacement" | dd of="$target_path" bs=1 count=1 conv=notrunc status=none + fi + ;; + size) + [[ ${1:-} = '--json' ]] || exit 64 + target_path=$(remote_path "${2:?}") + size_bytes=$(wc -c <"$target_path" | tr -d ' ') + printf '{"count":1,"bytes":%s}\n' "$size_bytes" + ;; + hashsum) + [[ ${1:-} = 'sha256' ]] || exit 64 + [[ ${WCTM_TEST_RCLONE_NATIVE_HASH:-available} = available ]] || exit 1 + target_path=$(remote_path "${2:?}") + sha256sum "$target_path" + ;; + cat) + target_path=$(remote_path "${1:?}") + cat -- "$target_path" + ;; + *) + echo >&2 'test rclone received an unsupported command' + exit 64 + ;; +esac diff --git a/scripts/ops/test-operations.sh b/scripts/ops/test-operations.sh new file mode 100755 index 0000000..34a07f2 --- /dev/null +++ b/scripts/ops/test-operations.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +image='postgres:16.15-alpine3.24@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685' +suffix="$(date -u '+%Y%m%dT%H%M%SZ')-$$" +source_container="wctm-ops-test-$suffix" +source_volume="wctm_ops_test_$suffix" +source_network="wctm_ops_test_$suffix" +workspace=$(mktemp -d) +secret_marker='test-password-must-not-appear' + +cleanup() { + docker rm --force "$source_container" >/dev/null 2>&1 || true + case "$source_volume" in + wctm_ops_test_*) docker volume rm "$source_volume" >/dev/null 2>&1 || true ;; + esac + case "$source_network" in + wctm_ops_test_*) docker network rm "$source_network" >/dev/null 2>&1 || true ;; + esac + rm -rf -- "$workspace" +} +trap cleanup EXIT HUP INT TERM + +for script in scripts/ops/*.sh; do + case "$script" in + *run-migrations.sh|*migrate-production.sh|*verify-runtime-role.sh|*deploy-production.sh) sh -n "$script" ;; + *) bash -n "$script" ;; + esac +done +bash -n scripts/ops/test-fixtures/rclone +docker compose config --quiet +docker compose build migrate >/dev/null +[[ $(docker run --rm --entrypoint id wctm-migrate:latest -u) = 1000 ]] + +if WCTM_MIGRATION_OPERATION=P7_2_EXPLICIT_MIGRATION DATABASE_URL='postgresql://wctm_runtime:sentinel@postgres/db' scripts/ops/run-migrations.sh >"$workspace/migration.log" 2>&1; then + echo >&2 'test failed: migration entrypoint accepted runtime identity' + exit 1 +fi +grep -q 'distinct PostgreSQL migration identity' "$workspace/migration.log" +! grep -q 'sentinel' "$workspace/migration.log" +if WCTM_MIGRATION_OPERATION=P7_2_EXPLICIT_MIGRATION DATABASE_URL='postgresql://wctm%5Fruntime:sentinel@postgres/db' scripts/ops/run-migrations.sh >"$workspace/migration-encoded.log" 2>&1; then + echo >&2 'test failed: migration entrypoint accepted encoded runtime identity' + exit 1 +fi +! grep -q 'sentinel' "$workspace/migration-encoded.log" + +mkdir "$workspace/retention" +for stamp in 20260101T000000Z 20260102T000000Z 20260103T000000Z; do + file="$workspace/retention/wctm-postgres-$stamp-deadbeef0000.dump" + printf '%s\n' "$stamp" >"$file" + printf '{}\n' >"$file.json" + (cd "$(dirname "$file")" && sha256sum "$(basename "$file")" >"$(basename "$file").sha256") +done +printf 'unrelated\n' >"$workspace/retention/do-not-delete.dump" +scripts/ops/backup-retention.sh --directory "$workspace/retention" --keep 1 >"$workspace/retention-dry.log" +[[ $(find "$workspace/retention" -name 'wctm-postgres-*.dump' | wc -l | tr -d ' ') = 3 ]] +scripts/ops/backup-retention.sh --directory "$workspace/retention" --keep 1 --apply >"$workspace/retention-apply.log" +[[ -f "$workspace/retention/wctm-postgres-20260103T000000Z-deadbeef0000.dump" ]] +[[ -f "$workspace/retention/do-not-delete.dump" ]] +[[ $(find "$workspace/retention" -name 'wctm-postgres-*.dump' | wc -l | tr -d ' ') = 1 ]] + +docker volume create "$source_volume" >/dev/null +docker network create "$source_network" >/dev/null +docker run -d --name "$source_container" --network "$source_network" --network-alias postgres \ + --mount "type=volume,source=$source_volume,destination=/var/lib/postgresql/data" \ + -e POSTGRES_DB=wctm_ops_test -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD="$secret_marker" \ + "$image" >/dev/null +stable_ready=0 +for _ in $(seq 1 60); do + if docker exec "$source_container" pg_isready -U postgres -d wctm_ops_test >/dev/null 2>&1; then + stable_ready=$((stable_ready + 1)) + ((stable_ready >= 3)) && break + else + stable_ready=0 + fi + sleep 1 +done +((stable_ready >= 3)) + +for attempt in 1 2; do + docker run --rm --network "$source_network" \ + -e WCTM_MIGRATION_OPERATION=P7_2_EXPLICIT_MIGRATION \ + -e "DATABASE_URL=postgresql://postgres:$secret_marker@postgres:5432/wctm_ops_test" \ + wctm-migrate:latest >"$workspace/migration-$attempt.log" 2>&1 + ! grep -q "$secret_marker" "$workspace/migration-$attempt.log" +done +[[ $(docker exec "$source_container" psql -XAtq -v ON_ERROR_STOP=1 -U postgres -d wctm_ops_test -c 'select count(*) from public._prisma_migrations where finished_at is not null and rolled_back_at is null') = 16 ]] + +docker exec -i "$source_container" psql -X -v ON_ERROR_STOP=1 -U postgres -d wctm_ops_test >/dev/null <<'SQL' +CREATE TABLE operations_probe (id integer PRIMARY KEY, value text NOT NULL); +INSERT INTO operations_probe VALUES (1, 'survives-backup-and-restore'); +SQL + +mkdir "$workspace/backups" +WCTM_POSTGRES_CONTAINER=$source_container scripts/ops/backup-postgres.sh --destination "$workspace/backups" >"$workspace/backup.log" 2>&1 +! grep -q "$secret_marker" "$workspace/backup.log" +backup=$(find "$workspace/backups" -maxdepth 1 -name 'wctm-postgres-*.dump' -print) +[[ -n "$backup" && -f "$backup.sha256" && -f "$backup.json" ]] + +mkdir "$workspace/test-bin" "$workspace/offsite-native" "$workspace/offsite-streamed" "$workspace/offsite-corrupt" +cp scripts/ops/test-fixtures/rclone "$workspace/test-bin/rclone" +chmod 0700 "$workspace/test-bin/rclone" +PATH="$workspace/test-bin:$PATH" scripts/ops/offsite-rclone.sh \ + "$backup" "$backup.sha256" "$backup.json" "testremote:$workspace/offsite-native" \ + >"$workspace/offsite-native.log" 2>&1 +grep -q 'content_sha256=verified method=native-sha256' "$workspace/offsite-native.log" +WCTM_TEST_RCLONE_NATIVE_HASH=unavailable PATH="$workspace/test-bin:$PATH" \ + scripts/ops/offsite-rclone.sh \ + "$backup" "$backup.sha256" "$backup.json" "testremote:$workspace/offsite-streamed" \ + >"$workspace/offsite-streamed.log" 2>&1 +grep -q 'content_sha256=verified method=streamed-sha256' "$workspace/offsite-streamed.log" +if WCTM_TEST_RCLONE_NATIVE_HASH=unavailable WCTM_TEST_RCLONE_CORRUPT_DUMP=true \ + PATH="$workspace/test-bin:$PATH" scripts/ops/offsite-rclone.sh \ + "$backup" "$backup.sha256" "$backup.json" "testremote:$workspace/offsite-corrupt" \ + >"$workspace/offsite-corrupt.log" 2>&1; then + echo >&2 'test failed: same-size remote corruption was accepted' + exit 1 +fi +remote_corrupt="$workspace/offsite-corrupt/$(basename "$backup")" +[[ $(wc -c <"$remote_corrupt" | tr -d ' ') = $(wc -c <"$backup" | tr -d ' ') ]] +[[ $(sha256sum "$remote_corrupt" | awk '{print $1}') != $(sha256sum "$backup" | awk '{print $1}') ]] +grep -q 'remote dump SHA-256 mismatch' "$workspace/offsite-corrupt.log" +! grep -q "$secret_marker" "$workspace/offsite-native.log" "$workspace/offsite-streamed.log" "$workspace/offsite-corrupt.log" + +if ! scripts/ops/restore-isolated.sh --backup "$backup" --critical-table tenants --critical-table stores --critical-table operations_probe >"$workspace/restore.log" 2>&1; then + cat "$workspace/restore.log" >&2 + exit 1 +fi +grep -q 'isolated restore: PASS migrations=16' "$workspace/restore.log" +grep -q 'critical table: tenants rows=0' "$workspace/restore.log" +grep -q 'critical table: stores rows=0' "$workspace/restore.log" +grep -q 'critical table: operations_probe rows=1' "$workspace/restore.log" +! grep -q "$secret_marker" "$workspace/restore.log" + +mkdir "$workspace/corrupt" +corrupt_backup="$workspace/corrupt/$(basename "$backup")" +cp "$backup" "$corrupt_backup" +cp "$backup.sha256" "$corrupt_backup.sha256" +cp "$backup.json" "$corrupt_backup.json" +printf 'corruption\n' >>"$corrupt_backup" +if scripts/ops/restore-isolated.sh --backup "$corrupt_backup" >"$workspace/corrupt.log" 2>&1; then + echo >&2 'test failed: corrupted backup was accepted' + exit 1 +fi + +echo 'operations integration: PASS migration-twice backup restore data checksum corruption retention offsite-native-sha256 offsite-streamed-sha256 offsite-same-size-corruption-rejected identity-boundary secret-safe-logs' diff --git a/scripts/ops/verify-runtime-role.sh b/scripts/ops/verify-runtime-role.sh new file mode 100755 index 0000000..ec963f9 --- /dev/null +++ b/scripts/ops/verify-runtime-role.sh @@ -0,0 +1,63 @@ +#!/bin/sh +set -eu + +container_id=$(docker compose ps -q backend) +[ -n "$container_id" ] || { + echo >&2 'runtime-role verification failed: backend container is not running' + exit 1 +} + +docker compose exec -T backend node <<'NODE' +const { Client } = require('pg'); + +async function main() { + const client = new Client({ connectionString: process.env.DATABASE_URL }); + await client.connect(); + try { + const result = await client.query(` + SELECT + current_user AS role_name, + r.rolsuper, + r.rolcreatedb, + r.rolcreaterole, + r.rolinherit, + r.rolreplication, + r.rolbypassrls, + has_schema_privilege(current_user, 'public', 'USAGE') AS schema_usage, + has_schema_privilege(current_user, 'public', 'CREATE') AS schema_create, + has_database_privilege(current_user, current_database(), 'TEMP') AS database_temp, + has_table_privilege(current_user, 'public._prisma_migrations', 'SELECT') OR + has_table_privilege(current_user, 'public._prisma_migrations', 'INSERT') OR + has_table_privilege(current_user, 'public._prisma_migrations', 'UPDATE') OR + has_table_privilege(current_user, 'public._prisma_migrations', 'DELETE') AS migration_table_dml, + (SELECT count(*)::int FROM pg_class WHERE relowner = r.oid) AS owned_objects + FROM pg_roles r + WHERE r.rolname = current_user + `); + const row = result.rows[0]; + const passed = row?.role_name === 'wctm_runtime' && + !row.rolsuper && !row.rolcreatedb && !row.rolcreaterole && + !row.rolinherit && !row.rolreplication && !row.rolbypassrls && + row.schema_usage && !row.schema_create && !row.database_temp && + !row.migration_table_dml && row.owned_objects === 0; + console.log(JSON.stringify({ + role: row?.role_name, + elevatedFlags: Boolean(row?.rolsuper || row?.rolcreatedb || row?.rolcreaterole || row?.rolinherit || row?.rolreplication || row?.rolbypassrls), + schemaUsage: row?.schema_usage, + schemaCreate: row?.schema_create, + databaseTemp: row?.database_temp, + migrationTableDml: row?.migration_table_dml, + ownedObjects: row?.owned_objects, + passed, + })); + if (!passed) process.exitCode = 1; + } finally { + await client.end(); + } +} + +main().catch(() => { + console.error('runtime-role verification failed'); + process.exitCode = 1; +}); +NODE