From bf09ba8bd7440e5caa61dee26bc6d234f0ea7969 Mon Sep 17 00:00:00 2001 From: ssavutu Date: Sat, 25 Jul 2026 03:29:25 -0400 Subject: [PATCH] fix(deploy): harden Delta Nginx bootstrap --- deploy/README.md | 38 ++++++- deploy/github-production.env.example | 2 +- ...triangle-cms-active-upstreams.conf.example | 2 +- deploy/nginx/triangle-cms.conf | 4 +- deploy/scripts/common.sh | 87 ++++++++++++++- deploy/scripts/deploy.sh | 2 +- deploy/scripts/deploy_scripts_test.sh | 103 +++++++++++++++++- deploy/scripts/rollback.sh | 2 +- 8 files changed, 223 insertions(+), 17 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index fcb7302..f58fe15 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -49,14 +49,39 @@ a manual server task: 3. Register a self-hosted GitHub Actions runner on Delta inside the Drexel VPN. Apply the labels `drexel-vpn`, `delta`, and `triangle-cms`. 4. Allow the runner user to run Docker and reload/test Nginx. Prefer narrow - sudoers rules for `nginx -t` and `nginx -s reload`. + sudoers rules only for `/usr/sbin/nginx -t` and + `/usr/sbin/nginx -s reload`. 5. Place the host-only production env file at the path configured by `DELTA_CMS_ENV_FILE`. Do not put it in git. 6. Install `nginx/triangle-cms.conf` as an enabled Nginx site. -7. Seed `/etc/nginx/triangle-cms-active-upstreams.conf` from the example include. +7. Create the narrow runtime-state directory and seed the active upstream + include: + + ```bash + sudo install -d \ + -o triangle-runner \ + -g triangle-runner \ + -m 0750 \ + /etc/nginx/triangle-cms + + sudo install \ + -o triangle-runner \ + -g triangle-runner \ + -m 0644 \ + deploy/nginx/triangle-cms-active-upstreams.conf.example \ + /etc/nginx/triangle-cms/active-upstreams.conf + ``` + 8. Validate Nginx and reload it once during bootstrap. 9. Confirm the runner can pull GHCR images and run `docker compose`. +Keep `/etc/nginx` root-owned and non-writable by the runner. Only +`/etc/nginx/triangle-cms` is writable runtime state for deployments, scoped to +the generated active upstream include. The directory should be owned by +`triangle-runner:triangle-runner` with mode `0750`; the active include should be +owned by `triangle-runner:triangle-runner` with mode `0644`. The host Nginx site +such as `/etc/nginx/sites-available/triangle-cms.conf` remains root-owned. + ## Required Host Environment Copy `cms.env.example` to the private host env path and fill it with real values. @@ -88,7 +113,7 @@ through the admin endpoint after deploys when needed. Configure these as GitHub Environment variables for `production`: - `DELTA_CMS_ENV_FILE` - absolute path to the host-only `cms.env`. -- `DELTA_NGINX_ACTIVE_INCLUDE` - usually `/etc/nginx/triangle-cms-active-upstreams.conf`. +- `DELTA_NGINX_ACTIVE_INCLUDE` - usually `/etc/nginx/triangle-cms/active-upstreams.conf`. - `DELTA_PUBLIC_BASE_URL` - initial HTTP VPN URL or hostname for smoke tests. Production database passwords, OIDC secrets, runner registration tokens, @@ -120,6 +145,7 @@ deploy/scripts/deploy.sh The script: - Acquires an exclusive `flock`. +- Runs deployment preflight checks before pulling images or starting containers. - Reads the active slot from the Nginx include. - Pulls the exact frontend/backend SHA images. - Starts only the inactive frontend/backend services. @@ -132,6 +158,12 @@ The script: It never runs `docker compose down -v` and never deletes persistent data. +The deployment preflight fails before pulling images, starting containers, or +switching Nginx if the active include directory is missing or not writable, an +existing active include is not readable and writable, the active slot is not +`blue` or `green`, `cms.env` is missing or unreadable, or Nginx validation/reload +privileges are not available. + ## Rollback Rollback switches Nginx back to the previous running slot: diff --git a/deploy/github-production.env.example b/deploy/github-production.env.example index 080d503..ab3442a 100644 --- a/deploy/github-production.env.example +++ b/deploy/github-production.env.example @@ -1,3 +1,3 @@ DELTA_CMS_ENV_FILE= -DELTA_NGINX_ACTIVE_INCLUDE= +DELTA_NGINX_ACTIVE_INCLUDE=/etc/nginx/triangle-cms/active-upstreams.conf DELTA_PUBLIC_BASE_URL= diff --git a/deploy/nginx/triangle-cms-active-upstreams.conf.example b/deploy/nginx/triangle-cms-active-upstreams.conf.example index d3bf968..fda8016 100644 --- a/deploy/nginx/triangle-cms-active-upstreams.conf.example +++ b/deploy/nginx/triangle-cms-active-upstreams.conf.example @@ -1,5 +1,5 @@ # Generated atomically by deploy/scripts/deploy.sh and deploy/scripts/rollback.sh. -# Copy to /etc/nginx/triangle-cms-active-upstreams.conf during bootstrap. +# Copy to /etc/nginx/triangle-cms/active-upstreams.conf during bootstrap. set $triangle_cms_slot blue; set $triangle_cms_frontend http://127.0.0.1:8091; set $triangle_cms_backend http://127.0.0.1:8081; diff --git a/deploy/nginx/triangle-cms.conf b/deploy/nginx/triangle-cms.conf index 32de964..659196d 100644 --- a/deploy/nginx/triangle-cms.conf +++ b/deploy/nginx/triangle-cms.conf @@ -12,7 +12,7 @@ server { listen 80; server_name _; - include /etc/nginx/triangle-cms-active-upstreams.conf; + include /etc/nginx/triangle-cms/active-upstreams.conf; proxy_http_version 1.1; proxy_set_header Host $host; @@ -22,7 +22,7 @@ server { location = /healthz { access_log off; - add_header Content-Type text/plain; + default_type text/plain; return 200 "ok\n"; } diff --git a/deploy/scripts/common.sh b/deploy/scripts/common.sh index fbbdc74..764c857 100755 --- a/deploy/scripts/common.sh +++ b/deploy/scripts/common.sh @@ -7,7 +7,8 @@ REPO_DIR="$(cd "${DEPLOY_DIR}/.." && pwd)" COMPOSE_FILE="${COMPOSE_FILE:-${DEPLOY_DIR}/compose.cms.yml}" ENV_FILE="${ENV_FILE:-${DEPLOY_DIR}/cms.env}" -NGINX_ACTIVE_INCLUDE="${NGINX_ACTIVE_INCLUDE:-/etc/nginx/triangle-cms-active-upstreams.conf}" +NGINX_ACTIVE_INCLUDE="${NGINX_ACTIVE_INCLUDE:-/etc/nginx/triangle-cms/active-upstreams.conf}" +NGINX_BIN="${NGINX_BIN:-/usr/sbin/nginx}" PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-http://127.0.0.1}" DEPLOY_LOCK_FILE="${DEPLOY_LOCK_FILE:-/tmp/triangle-cms-deploy.lock}" BACKEND_HEALTH_TIMEOUT="${BACKEND_HEALTH_TIMEOUT:-180}" @@ -26,6 +27,18 @@ require_file() { fi } +require_readable_file() { + local path="$1" + if [[ ! -f "${path}" ]]; then + echo "required file not found: ${path}" >&2 + return 1 + fi + if [[ ! -r "${path}" ]]; then + echo "required file is not readable: ${path}" >&2 + return 1 + fi +} + acquire_deploy_lock() { exec 9>"${DEPLOY_LOCK_FILE}" if ! flock -n 9; then @@ -122,9 +135,9 @@ nginx_test() { echo "NGINX_TEST_CMD/NGINX_RELOAD_CMD are test-only; set DEPLOY_TEST_MODE=1 outside production" >&2 return 2 elif command -v sudo >/dev/null 2>&1; then - sudo nginx -t + sudo -n "${NGINX_BIN}" -t else - nginx -t + "${NGINX_BIN}" -t fi } @@ -135,12 +148,76 @@ nginx_reload() { echo "NGINX_TEST_CMD/NGINX_RELOAD_CMD are test-only; set DEPLOY_TEST_MODE=1 outside production" >&2 return 2 elif command -v sudo >/dev/null 2>&1; then - sudo nginx -s reload + sudo -n "${NGINX_BIN}" -s reload else - nginx -s reload + "${NGINX_BIN}" -s reload fi } +nginx_reload_privilege_available() { + if [[ "${DEPLOY_TEST_MODE:-0}" == "1" && -n "${NGINX_RELOAD_CHECK_CMD:-}" ]]; then + bash -lc "${NGINX_RELOAD_CHECK_CMD}" + elif [[ -n "${NGINX_RELOAD_CHECK_CMD:-}" ]]; then + echo "NGINX_RELOAD_CHECK_CMD is test-only; set DEPLOY_TEST_MODE=1 outside production" >&2 + return 2 + elif command -v sudo >/dev/null 2>&1; then + sudo -n -l "${NGINX_BIN}" -s reload >/dev/null 2>&1 + else + [[ "$(id -u)" == "0" && -x "${NGINX_BIN}" ]] + fi +} + +deployment_preflight() { + local include_dir probe + include_dir="$(dirname "${NGINX_ACTIVE_INCLUDE}")" + + if [[ ! -d "${include_dir}" ]]; then + echo "deployment preflight failed: active include directory does not exist: ${include_dir}" >&2 + return 1 + fi + if [[ ! -w "${include_dir}" ]]; then + echo "deployment preflight failed: runner cannot write active include directory: ${include_dir}" >&2 + return 1 + fi + if ! probe="$(mktemp "${include_dir}/.triangle-cms-preflight.XXXXXX")"; then + echo "deployment preflight failed: runner cannot create files in active include directory: ${include_dir}" >&2 + return 1 + fi + rm -f "${probe}" + + if [[ -e "${NGINX_ACTIVE_INCLUDE}" ]]; then + if [[ ! -f "${NGINX_ACTIVE_INCLUDE}" ]]; then + echo "deployment preflight failed: active include is not a regular file: ${NGINX_ACTIVE_INCLUDE}" >&2 + return 1 + fi + if [[ ! -r "${NGINX_ACTIVE_INCLUDE}" ]]; then + echo "deployment preflight failed: active include is not readable: ${NGINX_ACTIVE_INCLUDE}" >&2 + return 1 + fi + if [[ ! -w "${NGINX_ACTIVE_INCLUDE}" ]]; then + echo "deployment preflight failed: active include is not writable: ${NGINX_ACTIVE_INCLUDE}" >&2 + return 1 + fi + fi + + if ! active_slot >/dev/null; then + echo "deployment preflight failed: active slot value is invalid" >&2 + return 1 + fi + if ! require_readable_file "${ENV_FILE}"; then + echo "deployment preflight failed: cms.env is missing or unreadable: ${ENV_FILE}" >&2 + return 1 + fi + if ! nginx_test; then + echo "deployment preflight failed: Nginx validation is not available or failed" >&2 + return 1 + fi + if ! nginx_reload_privilege_available; then + echo "deployment preflight failed: Nginx reload privilege is not available for ${NGINX_BIN} -s reload" >&2 + return 1 + fi +} + http_get() { local url="$1" if command -v curl >/dev/null 2>&1; then diff --git a/deploy/scripts/deploy.sh b/deploy/scripts/deploy.sh index 00dda00..dff1141 100755 --- a/deploy/scripts/deploy.sh +++ b/deploy/scripts/deploy.sh @@ -20,8 +20,8 @@ fi export CMS_IMAGE_TAG require_file "${COMPOSE_FILE}" -require_file "${ENV_FILE}" acquire_deploy_lock +deployment_preflight current_slot="$(active_slot)" validate_slot "${current_slot}" diff --git a/deploy/scripts/deploy_scripts_test.sh b/deploy/scripts/deploy_scripts_test.sh index 4c72e9a..bcaa1d0 100755 --- a/deploy/scripts/deploy_scripts_test.sh +++ b/deploy/scripts/deploy_scripts_test.sh @@ -22,6 +22,15 @@ assert_file_contains() { grep -q "${pattern}" "${path}" || fail "expected ${path} to contain ${pattern}" } +assert_file_not_contains() { + local path="$1" + local pattern="$2" + [[ -f "${path}" ]] || fail "expected file to exist: ${path}" + if grep -q "${pattern}" "${path}"; then + fail "expected ${path} not to contain ${pattern}" + fi +} + assert_no_switch_temps() { local dir="$1" if find "${dir}" -maxdepth 1 -type d -name 'triangle-cms-switch.*' | grep -q .; then @@ -29,11 +38,15 @@ assert_no_switch_temps() { fi } +assert_no_docker_invocations() { + [[ ! -s "${FAKE_DOCKER_LOG}" ]] || fail "expected no docker invocations" +} + make_case() { local name="$1" local dir="${TEST_ROOT}/${name}" - mkdir -p "${dir}/bin" - NGINX_ACTIVE_INCLUDE="${dir}/active.conf" + mkdir -p "${dir}/bin" "${dir}/triangle-cms" + NGINX_ACTIVE_INCLUDE="${dir}/triangle-cms/active-upstreams.conf" ENV_FILE="${dir}/cms.env" COMPOSE_FILE="${dir}/compose.yml" DEPLOY_LOCK_FILE="${dir}/deploy.lock" @@ -44,13 +57,16 @@ make_case() { DEPLOY_TEST_MODE=1 NGINX_TEST_CMD='exit "${FAKE_NGINX_TEST_STATUS:-0}"' NGINX_RELOAD_CMD='exit "${FAKE_NGINX_RELOAD_STATUS:-0}"' + NGINX_RELOAD_CHECK_CMD='exit "${FAKE_NGINX_RELOAD_CHECK_STATUS:-0}"' FAKE_NGINX_TEST_STATUS=0 FAKE_NGINX_RELOAD_STATUS=0 + FAKE_NGINX_RELOAD_CHECK_STATUS=0 FAIL_READINESS=0 FAIL_PUBLIC=0 export NGINX_ACTIVE_INCLUDE ENV_FILE COMPOSE_FILE DEPLOY_LOCK_FILE PUBLIC_BASE_URL export BACKEND_HEALTH_TIMEOUT FRONTEND_HEALTH_TIMEOUT PUBLIC_HEALTH_TIMEOUT - export DEPLOY_TEST_MODE NGINX_TEST_CMD NGINX_RELOAD_CMD FAKE_NGINX_TEST_STATUS FAKE_NGINX_RELOAD_STATUS + export DEPLOY_TEST_MODE NGINX_TEST_CMD NGINX_RELOAD_CMD NGINX_RELOAD_CHECK_CMD + export FAKE_NGINX_TEST_STATUS FAKE_NGINX_RELOAD_STATUS FAKE_NGINX_RELOAD_CHECK_STATUS export FAIL_READINESS FAIL_PUBLIC : > "${ENV_FILE}" : > "${COMPOSE_FILE}" @@ -84,6 +100,16 @@ EOF export PATH } +run_deploy_expect_failure_without_docker() { + local sha + sha="0123456789abcdef0123456789abcdef01234567" + write_fake_bin "${CASE_DIR}" + if "${SCRIPT_DIR}/deploy.sh" "${sha}"; then + fail "expected deploy preflight failure" + fi + assert_no_docker_invocations +} + test_first_deployment_switch() { local dir make_case first @@ -145,6 +171,54 @@ test_reload_failure_restores_include() { assert_no_switch_temps "${dir}" } +test_missing_include_directory_preflight_fails_before_pull() { + local dir + make_case missing_include_dir + dir="${CASE_DIR}" + NGINX_ACTIVE_INCLUDE="${dir}/missing/active-upstreams.conf" + export NGINX_ACTIVE_INCLUDE + run_deploy_expect_failure_without_docker +} + +test_non_writable_include_directory_preflight_fails_before_pull() { + local dir + make_case non_writable_include_dir + dir="${CASE_DIR}" + chmod 0500 "${dir}/triangle-cms" + run_deploy_expect_failure_without_docker + chmod 0750 "${dir}/triangle-cms" +} + +test_existing_non_writable_include_preflight_fails_before_pull() { + local dir + make_case non_writable_include + dir="${CASE_DIR}" + write_include_file blue "${NGINX_ACTIVE_INCLUDE}" + chmod 0444 "${NGINX_ACTIVE_INCLUDE}" + run_deploy_expect_failure_without_docker + chmod 0644 "${NGINX_ACTIVE_INCLUDE}" +} + +test_invalid_active_slot_preflight_fails_before_pull() { + make_case invalid_active_slot + echo 'set $triangle_cms_slot purple;' > "${NGINX_ACTIVE_INCLUDE}" + run_deploy_expect_failure_without_docker +} + +test_successful_deployment_preflight() { + make_case successful_preflight + write_include_file blue "${NGINX_ACTIVE_INCLUDE}" + deployment_preflight +} + +test_nginx_reload_privilege_preflight_fails_before_pull() { + make_case reload_privilege + write_include_file blue "${NGINX_ACTIVE_INCLUDE}" + FAKE_NGINX_RELOAD_CHECK_STATUS=1 + export FAKE_NGINX_RELOAD_CHECK_STATUS + run_deploy_expect_failure_without_docker +} + test_failed_readiness_leaves_active_slot() { local dir sha make_case readiness @@ -203,14 +277,37 @@ test_lock_contention() { exec 8>&- } +test_host_nginx_health_uses_default_type() { + local nginx_conf="${SCRIPT_DIR}/../nginx/triangle-cms.conf" + assert_file_contains "${nginx_conf}" 'default_type text/plain' + assert_file_not_contains "${nginx_conf}" 'add_header Content-Type text/plain' +} + +test_no_old_production_include_path_references() { + local old_path + old_path="/etc/nginx/triangle-cms" + old_path="${old_path}-active-upstreams.conf" + if git grep -n -- "${old_path}" -- .; then + fail "found old production include path reference" + fi +} + test_first_deployment_switch test_blue_to_green_switch test_malformed_active_include_fails test_nginx_test_failure_restores_include test_reload_failure_restores_include +test_missing_include_directory_preflight_fails_before_pull +test_non_writable_include_directory_preflight_fails_before_pull +test_existing_non_writable_include_preflight_fails_before_pull +test_invalid_active_slot_preflight_fails_before_pull +test_successful_deployment_preflight +test_nginx_reload_privilege_preflight_fails_before_pull test_failed_readiness_leaves_active_slot test_failed_public_smoke_rolls_back test_invalid_and_malicious_sha test_lock_contention +test_host_nginx_health_uses_default_type +test_no_old_production_include_path_references echo "deploy script tests passed" diff --git a/deploy/scripts/rollback.sh b/deploy/scripts/rollback.sh index f7a52cb..06cc812 100755 --- a/deploy/scripts/rollback.sh +++ b/deploy/scripts/rollback.sh @@ -4,8 +4,8 @@ set -Eeuo pipefail source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh" require_file "${COMPOSE_FILE}" -require_file "${ENV_FILE}" acquire_deploy_lock +deployment_preflight current_slot="$(active_slot)" validate_slot "${current_slot}"