diff --git a/docs/operations/P7_2_P7_3_RUNBOOK.md b/docs/operations/P7_2_P7_3_RUNBOOK.md index b444b21..0beb7e3 100644 --- a/docs/operations/P7_2_P7_3_RUNBOOK.md +++ b/docs/operations/P7_2_P7_3_RUNBOOK.md @@ -106,10 +106,20 @@ in synchronized `origin/main`, valid Compose configuration, running services, an all-PASS production configuration audit, and a fresh verified backup. It builds the migration/backend/bot images, runs migration before cutover, stops immediately on migration failure, recreates backend then bot, checks local -health/readiness, and verifies the actual backend identity and restrictions. +health/readiness with bounded startup retries, and verifies the actual backend +identity and restrictions. The two local probes must both pass in the same +attempt; after at most 12 attempts, with 1, 2, 4, then 5-second capped delays, +the script fails and names the last failing probe. It does not reconcile PostgreSQL/Redis images or run product smoke by itself; those controlled gates below surround the application deployment. +Run every production backend configuration audit directly through the compiled +CLI in the minimized image: + +```bash +docker compose exec -T backend node dist/security/security-config-audit.cli.js +``` + ## Controlled PostgreSQL/Redis image reconciliation The repository requires these exact references: diff --git a/scripts/ops/deploy-production.sh b/scripts/ops/deploy-production.sh index 509d264..ecbfb0b 100755 --- a/scripts/ops/deploy-production.sh +++ b/scripts/ops/deploy-production.sh @@ -44,7 +44,7 @@ docker compose config --quiet exit 1 } if [ -n "$(docker compose ps -q backend)" ]; then - docker compose exec -T backend npm run security:config-audit + docker compose exec -T backend node dist/security/security-config-audit.cli.js else echo 'pre-cutover config audit: previously required by Gate 1 (backend currently stopped)' fi @@ -59,8 +59,30 @@ docker compose up -d --no-deps --force-recreate backend docker compose up -d --no-deps --force-recreate telegram-bot runtime_port=${PORT:-3000} -curl --fail --silent --show-error "http://127.0.0.1:$runtime_port/api/health" >/dev/null -curl --fail --silent --show-error "http://127.0.0.1:$runtime_port/api/health/readiness" >/dev/null +attempt=1 +max_attempts=12 +retry_delay=1 +while :; do + if curl --fail --silent --connect-timeout 1 --max-time 2 "http://127.0.0.1:$runtime_port/api/health" >/dev/null 2>&1; then + if curl --fail --silent --connect-timeout 1 --max-time 2 "http://127.0.0.1:$runtime_port/api/health/readiness" >/dev/null 2>&1; then + break + fi + failed_check=readiness + else + failed_check=health + fi + + if [ "$attempt" -ge "$max_attempts" ]; then + echo >&2 "deployment failed: backend $failed_check did not pass after $max_attempts attempts" + exit 1 + fi + sleep "$retry_delay" + attempt=$((attempt + 1)) + if [ "$retry_delay" -lt 5 ]; then + retry_delay=$((retry_delay * 2)) + [ "$retry_delay" -le 5 ] || retry_delay=5 + fi +done scripts/ops/verify-runtime-role.sh echo "deployment path: PASS revision=$actual_revision" diff --git a/scripts/ops/test-deploy-production.sh b/scripts/ops/test-deploy-production.sh new file mode 100755 index 0000000..fac22e7 --- /dev/null +++ b/scripts/ops/test-deploy-production.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +workspace=$(mktemp -d) +trap 'rm -rf -- "$workspace"' EXIT +mkdir -p "$workspace/scripts/ops" "$workspace/bin" "$workspace/state" +cp scripts/ops/deploy-production.sh "$workspace/scripts/ops/deploy-production.sh" + +cat >"$workspace/bin/git" <<'SH' +#!/bin/sh +case "$*" in + 'rev-parse HEAD') printf '%s\n' "$TEST_REVISION" ;; + 'status --porcelain'|"merge-base --is-ancestor $TEST_REVISION origin/main") ;; + *) exit 99 ;; +esac +SH + +cat >"$workspace/bin/docker" <<'SH' +#!/bin/sh +printf '%s\n' "$*" >>"$TEST_STATE/docker.log" +case "$*" in + 'compose config --quiet'|'compose build migrate backend telegram-bot'|\ + 'compose up -d --no-deps --force-recreate backend'|\ + 'compose up -d --no-deps --force-recreate telegram-bot'|\ + 'compose exec -T backend node dist/security/security-config-audit.cli.js') ;; + 'compose ps --status running -q postgres'|'compose ps --status running -q redis'|\ + 'compose ps -q backend') printf 'test-container-id\n' ;; + *) exit 99 ;; +esac +SH + +cat >"$workspace/bin/curl" <<'SH' +#!/bin/sh +count=$(cat "$TEST_STATE/curl-count") +count=$((count + 1)) +printf '%s\n' "$count" >"$TEST_STATE/curl-count" +printf '%s\n' "$*" >>"$TEST_STATE/curl.log" +for arg do url=$arg; done +if [ "$TEST_CURL_MODE" = delayed ] && [ "$count" -eq 1 ]; then + exit 56 +fi +if [ "$TEST_CURL_MODE" = persistent ] && [ "${url##*/}" = readiness ]; then + exit 22 +fi +exit 0 +SH + +cat >"$workspace/bin/sleep" <<'SH' +#!/bin/sh +printf '%s\n' "$*" >>"$TEST_STATE/sleep.log" +SH + +for helper in backup-postgres migrate-production verify-runtime-role; do + cat >"$workspace/scripts/ops/$helper.sh" <<'SH' +#!/bin/sh +printf '%s\n' "${0##*/}" >>"$TEST_STATE/operations.log" +SH +done +chmod +x "$workspace"/bin/* "$workspace"/scripts/ops/*.sh + +revision=1234567890abcdef1234567890abcdef12345678 +export TEST_REVISION=$revision TEST_STATE="$workspace/state" +export PATH="$workspace/bin:$PATH" + +run_deploy() { + (cd "$workspace" && sh scripts/ops/deploy-production.sh \ + --revision "$revision" \ + --migration-credential-file /unused/test-migration-file \ + --backup-directory /unused/test-backups) +} + +printf '0\n' >"$TEST_STATE/curl-count" +export TEST_CURL_MODE=delayed +run_deploy >"$workspace/delayed.log" 2>&1 +grep -q 'deployment path: PASS' "$workspace/delayed.log" +[[ $(cat "$TEST_STATE/curl-count") = 3 ]] +[[ $(cat "$TEST_STATE/sleep.log") = 1 ]] +grep -qx 'compose exec -T backend node dist/security/security-config-audit.cli.js' "$TEST_STATE/docker.log" +! grep -q 'compose exec -T backend npm ' "$TEST_STATE/docker.log" +grep -qx 'verify-runtime-role.sh' "$TEST_STATE/operations.log" + +rm -f "$TEST_STATE"/*.log +printf '0\n' >"$TEST_STATE/curl-count" +export TEST_CURL_MODE=persistent +if run_deploy >"$workspace/persistent.log" 2>&1; then + echo >&2 'test failed: persistent readiness failure was accepted' + exit 1 +fi +grep -q 'deployment failed: backend readiness did not pass after 12 attempts' "$workspace/persistent.log" +[[ $(cat "$TEST_STATE/curl-count") = 24 ]] +[[ $(wc -l <"$TEST_STATE/sleep.log" | tr -d ' ') = 11 ]] +[[ $(sed -n '1,4p' "$TEST_STATE/sleep.log" | tr '\n' ',') = '1,2,4,5,' ]] +! grep -q 'verify-runtime-role.sh' "$TEST_STATE/operations.log" + +echo 'deployment script: PASS startup-reset-retry persistent-readiness-failure compiled-config-audit'