Delta Nginx bootstrap

deploy/README.md

@@ -49,14 +49,39 @@ a manual server task:
 3. Register a self-hosted GitHub Actions runner on Delta inside the Drexel VPN.
    Apply the labels `drexel-vpn`, `delta`, and `triangle-cms`.
 4. Allow the runner user to run Docker and reload/test Nginx. Prefer narrow
-   sudoers rules for `nginx -t` and `nginx -s reload`.
+   sudoers rules only for `/usr/sbin/nginx -t` and
+   `/usr/sbin/nginx -s reload`.
 5. Place the host-only production env file at the path configured by
    `DELTA_CMS_ENV_FILE`. Do not put it in git.
 6. Install `nginx/triangle-cms.conf` as an enabled Nginx site.
-7. Seed `/etc/nginx/triangle-cms-active-upstreams.conf` from the example include.
+7. Create the narrow runtime-state directory and seed the active upstream
+   include:
+
+   ```bash
+   sudo install -d \
+     -o triangle-runner \
+     -g triangle-runner \
+     -m 0750 \
+     /etc/nginx/triangle-cms
+
+   sudo install \
+     -o triangle-runner \
+     -g triangle-runner \
+     -m 0644 \
+     deploy/nginx/triangle-cms-active-upstreams.conf.example \
+     /etc/nginx/triangle-cms/active-upstreams.conf
+   ```
+
 8. Validate Nginx and reload it once during bootstrap.
 9. Confirm the runner can pull GHCR images and run `docker compose`.
 
+Keep `/etc/nginx` root-owned and non-writable by the runner. Only
+`/etc/nginx/triangle-cms` is writable runtime state for deployments, scoped to
+the generated active upstream include. The directory should be owned by
+`triangle-runner:triangle-runner` with mode `0750`; the active include should be
+owned by `triangle-runner:triangle-runner` with mode `0644`. The host Nginx site
+such as `/etc/nginx/sites-available/triangle-cms.conf` remains root-owned.
+
 ## Required Host Environment
 
 Copy `cms.env.example` to the private host env path and fill it with real values.
@@ -88,7 +113,7 @@ through the admin endpoint after deploys when needed.
 Configure these as GitHub Environment variables for `production`:
 
 - `DELTA_CMS_ENV_FILE` - absolute path to the host-only `cms.env`.
-- `DELTA_NGINX_ACTIVE_INCLUDE` - usually `/etc/nginx/triangle-cms-active-upstreams.conf`.
+- `DELTA_NGINX_ACTIVE_INCLUDE` - usually `/etc/nginx/triangle-cms/active-upstreams.conf`.
 - `DELTA_PUBLIC_BASE_URL` - initial HTTP VPN URL or hostname for smoke tests.
 
 Production database passwords, OIDC secrets, runner registration tokens,
@@ -120,6 +145,7 @@ deploy/scripts/deploy.sh <full-commit-sha>
 The script:
 
 - Acquires an exclusive `flock`.
+- Runs deployment preflight checks before pulling images or starting containers.
 - Reads the active slot from the Nginx include.
 - Pulls the exact frontend/backend SHA images.
 - Starts only the inactive frontend/backend services.
@@ -132,6 +158,12 @@ The script:
 
 It never runs `docker compose down -v` and never deletes persistent data.
 
+The deployment preflight fails before pulling images, starting containers, or
+switching Nginx if the active include directory is missing or not writable, an
+existing active include is not readable and writable, the active slot is not
+`blue` or `green`, `cms.env` is missing or unreadable, or Nginx validation/reload
+privileges are not available.
+
 ## Rollback
 
 Rollback switches Nginx back to the previous running slot:

deploy/github-production.env.example

@@ -1,3 +1,3 @@
 DELTA_CMS_ENV_FILE=
-DELTA_NGINX_ACTIVE_INCLUDE=
+DELTA_NGINX_ACTIVE_INCLUDE=/etc/nginx/triangle-cms/active-upstreams.conf
 DELTA_PUBLIC_BASE_URL=

deploy/nginx/triangle-cms-active-upstreams.conf.example

@@ -1,5 +1,5 @@
 # Generated atomically by deploy/scripts/deploy.sh and deploy/scripts/rollback.sh.
-# Copy to /etc/nginx/triangle-cms-active-upstreams.conf during bootstrap.
+# Copy to /etc/nginx/triangle-cms/active-upstreams.conf during bootstrap.
 set $triangle_cms_slot blue;
 set $triangle_cms_frontend http://127.0.0.1:8091;
 set $triangle_cms_backend http://127.0.0.1:8081;

deploy/nginx/triangle-cms.conf

@@ -12,7 +12,7 @@ server {
     listen 80;
     server_name _;
 
-    include /etc/nginx/triangle-cms-active-upstreams.conf;
+    include /etc/nginx/triangle-cms/active-upstreams.conf;
 
     proxy_http_version 1.1;
     proxy_set_header Host $host;
@@ -22,7 +22,7 @@ server {
 
     location = /healthz {
         access_log off;
-        add_header Content-Type text/plain;
+        default_type text/plain;
         return 200 "ok\n";
     }
 

deploy/scripts/common.sh

@@ -7,7 +7,8 @@ REPO_DIR="$(cd "${DEPLOY_DIR}/.." && pwd)"
 
 COMPOSE_FILE="${COMPOSE_FILE:-${DEPLOY_DIR}/compose.cms.yml}"
 ENV_FILE="${ENV_FILE:-${DEPLOY_DIR}/cms.env}"
-NGINX_ACTIVE_INCLUDE="${NGINX_ACTIVE_INCLUDE:-/etc/nginx/triangle-cms-active-upstreams.conf}"
+NGINX_ACTIVE_INCLUDE="${NGINX_ACTIVE_INCLUDE:-/etc/nginx/triangle-cms/active-upstreams.conf}"
+NGINX_BIN="${NGINX_BIN:-/usr/sbin/nginx}"
 PUBLIC_BASE_URL="${PUBLIC_BASE_URL:-http://127.0.0.1}"
 DEPLOY_LOCK_FILE="${DEPLOY_LOCK_FILE:-/tmp/triangle-cms-deploy.lock}"
 BACKEND_HEALTH_TIMEOUT="${BACKEND_HEALTH_TIMEOUT:-180}"
@@ -26,6 +27,18 @@ require_file() {
   fi
 }
 
+require_readable_file() {
+	local path="$1"
+	if [[ ! -f "${path}" ]]; then
+		echo "required file not found: ${path}" >&2
+		return 1
+	fi
+	if [[ ! -r "${path}" ]]; then
+		echo "required file is not readable: ${path}" >&2
+		return 1
+	fi
+}
+
 acquire_deploy_lock() {
   exec 9>"${DEPLOY_LOCK_FILE}"
   if ! flock -n 9; then
@@ -122,9 +135,9 @@ nginx_test() {
 		echo "NGINX_TEST_CMD/NGINX_RELOAD_CMD are test-only; set DEPLOY_TEST_MODE=1 outside production" >&2
 		return 2
 	elif command -v sudo >/dev/null 2>&1; then
-		sudo nginx -t
+		sudo -n "${NGINX_BIN}" -t
 	else
-    nginx -t
+		"${NGINX_BIN}" -t
   fi
 }
 
@@ -135,12 +148,76 @@ nginx_reload() {
 		echo "NGINX_TEST_CMD/NGINX_RELOAD_CMD are test-only; set DEPLOY_TEST_MODE=1 outside production" >&2
 		return 2
 	elif command -v sudo >/dev/null 2>&1; then
-		sudo nginx -s reload
+		sudo -n "${NGINX_BIN}" -s reload
 	else
-    nginx -s reload
+		"${NGINX_BIN}" -s reload
   fi
 }
 
+nginx_reload_privilege_available() {
+	if [[ "${DEPLOY_TEST_MODE:-0}" == "1" && -n "${NGINX_RELOAD_CHECK_CMD:-}" ]]; then
+		bash -lc "${NGINX_RELOAD_CHECK_CMD}"
+	elif [[ -n "${NGINX_RELOAD_CHECK_CMD:-}" ]]; then
+		echo "NGINX_RELOAD_CHECK_CMD is test-only; set DEPLOY_TEST_MODE=1 outside production" >&2
+		return 2
+	elif command -v sudo >/dev/null 2>&1; then
+		sudo -n -l "${NGINX_BIN}" -s reload >/dev/null 2>&1
+	else
+		[[ "$(id -u)" == "0" && -x "${NGINX_BIN}" ]]
+	fi
+}
+
+deployment_preflight() {
+	local include_dir probe
+	include_dir="$(dirname "${NGINX_ACTIVE_INCLUDE}")"
+
+	if [[ ! -d "${include_dir}" ]]; then
+		echo "deployment preflight failed: active include directory does not exist: ${include_dir}" >&2
+		return 1
+	fi
+	if [[ ! -w "${include_dir}" ]]; then
+		echo "deployment preflight failed: runner cannot write active include directory: ${include_dir}" >&2
+		return 1
+	fi
+	if ! probe="$(mktemp "${include_dir}/.triangle-cms-preflight.XXXXXX")"; then
+		echo "deployment preflight failed: runner cannot create files in active include directory: ${include_dir}" >&2
+		return 1
+	fi
+	rm -f "${probe}"
+
+	if [[ -e "${NGINX_ACTIVE_INCLUDE}" ]]; then
+		if [[ ! -f "${NGINX_ACTIVE_INCLUDE}" ]]; then
+			echo "deployment preflight failed: active include is not a regular file: ${NGINX_ACTIVE_INCLUDE}" >&2
+			return 1
+		fi
+		if [[ ! -r "${NGINX_ACTIVE_INCLUDE}" ]]; then
+			echo "deployment preflight failed: active include is not readable: ${NGINX_ACTIVE_INCLUDE}" >&2
+			return 1
+		fi
+		if [[ ! -w "${NGINX_ACTIVE_INCLUDE}" ]]; then
+			echo "deployment preflight failed: active include is not writable: ${NGINX_ACTIVE_INCLUDE}" >&2
+			return 1
+		fi
+	fi
+
+	if ! active_slot >/dev/null; then
+		echo "deployment preflight failed: active slot value is invalid" >&2
+		return 1
+	fi
+	if ! require_readable_file "${ENV_FILE}"; then
+		echo "deployment preflight failed: cms.env is missing or unreadable: ${ENV_FILE}" >&2
+		return 1
+	fi
+	if ! nginx_test; then
+		echo "deployment preflight failed: Nginx validation is not available or failed" >&2
+		return 1
+	fi
+	if ! nginx_reload_privilege_available; then
+		echo "deployment preflight failed: Nginx reload privilege is not available for ${NGINX_BIN} -s reload" >&2
+		return 1
+	fi
+}
+
 http_get() {
   local url="$1"
   if command -v curl >/dev/null 2>&1; then

deploy/scripts/deploy.sh

@@ -20,8 +20,8 @@ fi
 export CMS_IMAGE_TAG
 
 require_file "${COMPOSE_FILE}"
-require_file "${ENV_FILE}"
 acquire_deploy_lock
+deployment_preflight
 
 current_slot="$(active_slot)"
 validate_slot "${current_slot}"

deploy/scripts/deploy_scripts_test.sh

@@ -22,18 +22,31 @@ assert_file_contains() {
   grep -q "${pattern}" "${path}" || fail "expected ${path} to contain ${pattern}"
 }
 
+assert_file_not_contains() {
+  local path="$1"
+  local pattern="$2"
+  [[ -f "${path}" ]] || fail "expected file to exist: ${path}"
+  if grep -q "${pattern}" "${path}"; then
+    fail "expected ${path} not to contain ${pattern}"
+  fi
+}
+
 assert_no_switch_temps() {
   local dir="$1"
   if find "${dir}" -maxdepth 1 -type d -name 'triangle-cms-switch.*' | grep -q .; then
     fail "switch temp directory was not cleaned up in ${dir}"
   fi
 }
 
+assert_no_docker_invocations() {
+  [[ ! -s "${FAKE_DOCKER_LOG}" ]] || fail "expected no docker invocations"
+}
+
 make_case() {
   local name="$1"
   local dir="${TEST_ROOT}/${name}"
-  mkdir -p "${dir}/bin"
-  NGINX_ACTIVE_INCLUDE="${dir}/active.conf"
+  mkdir -p "${dir}/bin" "${dir}/triangle-cms"
+  NGINX_ACTIVE_INCLUDE="${dir}/triangle-cms/active-upstreams.conf"
   ENV_FILE="${dir}/cms.env"
   COMPOSE_FILE="${dir}/compose.yml"
   DEPLOY_LOCK_FILE="${dir}/deploy.lock"
@@ -44,13 +57,16 @@ make_case() {
   DEPLOY_TEST_MODE=1
   NGINX_TEST_CMD='exit "${FAKE_NGINX_TEST_STATUS:-0}"'
   NGINX_RELOAD_CMD='exit "${FAKE_NGINX_RELOAD_STATUS:-0}"'
+  NGINX_RELOAD_CHECK_CMD='exit "${FAKE_NGINX_RELOAD_CHECK_STATUS:-0}"'
   FAKE_NGINX_TEST_STATUS=0
   FAKE_NGINX_RELOAD_STATUS=0
+  FAKE_NGINX_RELOAD_CHECK_STATUS=0
   FAIL_READINESS=0
   FAIL_PUBLIC=0
   export NGINX_ACTIVE_INCLUDE ENV_FILE COMPOSE_FILE DEPLOY_LOCK_FILE PUBLIC_BASE_URL
   export BACKEND_HEALTH_TIMEOUT FRONTEND_HEALTH_TIMEOUT PUBLIC_HEALTH_TIMEOUT
-  export DEPLOY_TEST_MODE NGINX_TEST_CMD NGINX_RELOAD_CMD FAKE_NGINX_TEST_STATUS FAKE_NGINX_RELOAD_STATUS
+  export DEPLOY_TEST_MODE NGINX_TEST_CMD NGINX_RELOAD_CMD NGINX_RELOAD_CHECK_CMD
+  export FAKE_NGINX_TEST_STATUS FAKE_NGINX_RELOAD_STATUS FAKE_NGINX_RELOAD_CHECK_STATUS
   export FAIL_READINESS FAIL_PUBLIC
   : > "${ENV_FILE}"
   : > "${COMPOSE_FILE}"
@@ -84,6 +100,16 @@ EOF
   export PATH
 }
 
+run_deploy_expect_failure_without_docker() {
+  local sha
+  sha="0123456789abcdef0123456789abcdef01234567"
+  write_fake_bin "${CASE_DIR}"
+  if "${SCRIPT_DIR}/deploy.sh" "${sha}"; then
+    fail "expected deploy preflight failure"
+  fi
+  assert_no_docker_invocations
+}
+
 test_first_deployment_switch() {
   local dir
   make_case first
@@ -145,6 +171,54 @@ test_reload_failure_restores_include() {
   assert_no_switch_temps "${dir}"
 }
 
+test_missing_include_directory_preflight_fails_before_pull() {
+  local dir
+  make_case missing_include_dir
+  dir="${CASE_DIR}"
+  NGINX_ACTIVE_INCLUDE="${dir}/missing/active-upstreams.conf"
+  export NGINX_ACTIVE_INCLUDE
+  run_deploy_expect_failure_without_docker
+}
+
+test_non_writable_include_directory_preflight_fails_before_pull() {
+  local dir
+  make_case non_writable_include_dir
+  dir="${CASE_DIR}"
+  chmod 0500 "${dir}/triangle-cms"
+  run_deploy_expect_failure_without_docker
+  chmod 0750 "${dir}/triangle-cms"
+}
+
+test_existing_non_writable_include_preflight_fails_before_pull() {
+  local dir
+  make_case non_writable_include
+  dir="${CASE_DIR}"
+  write_include_file blue "${NGINX_ACTIVE_INCLUDE}"
+  chmod 0444 "${NGINX_ACTIVE_INCLUDE}"
+  run_deploy_expect_failure_without_docker
+  chmod 0644 "${NGINX_ACTIVE_INCLUDE}"
+}
+
+test_invalid_active_slot_preflight_fails_before_pull() {
+  make_case invalid_active_slot
+  echo 'set $triangle_cms_slot purple;' > "${NGINX_ACTIVE_INCLUDE}"
+  run_deploy_expect_failure_without_docker
+}
+
+test_successful_deployment_preflight() {
+  make_case successful_preflight
+  write_include_file blue "${NGINX_ACTIVE_INCLUDE}"
+  deployment_preflight
+}
+
+test_nginx_reload_privilege_preflight_fails_before_pull() {
+  make_case reload_privilege
+  write_include_file blue "${NGINX_ACTIVE_INCLUDE}"
+  FAKE_NGINX_RELOAD_CHECK_STATUS=1
+  export FAKE_NGINX_RELOAD_CHECK_STATUS
+  run_deploy_expect_failure_without_docker
+}
+
 test_failed_readiness_leaves_active_slot() {
   local dir sha
   make_case readiness
@@ -203,14 +277,37 @@ test_lock_contention() {
   exec 8>&-
 }
 
+test_host_nginx_health_uses_default_type() {
+  local nginx_conf="${SCRIPT_DIR}/../nginx/triangle-cms.conf"
+  assert_file_contains "${nginx_conf}" 'default_type text/plain'
+  assert_file_not_contains "${nginx_conf}" 'add_header Content-Type text/plain'
+}
+
+test_no_old_production_include_path_references() {
+  local old_path
+  old_path="/etc/nginx/triangle-cms"
+  old_path="${old_path}-active-upstreams.conf"
+  if git grep -n -- "${old_path}" -- .; then
+    fail "found old production include path reference"
+  fi
+}
+
 test_first_deployment_switch
 test_blue_to_green_switch
 test_malformed_active_include_fails
 test_nginx_test_failure_restores_include
 test_reload_failure_restores_include
+test_missing_include_directory_preflight_fails_before_pull
+test_non_writable_include_directory_preflight_fails_before_pull
+test_existing_non_writable_include_preflight_fails_before_pull
+test_invalid_active_slot_preflight_fails_before_pull
+test_successful_deployment_preflight
+test_nginx_reload_privilege_preflight_fails_before_pull
 test_failed_readiness_leaves_active_slot
 test_failed_public_smoke_rolls_back
 test_invalid_and_malicious_sha
 test_lock_contention
+test_host_nginx_health_uses_default_type
+test_no_old_production_include_path_references
 
 echo "deploy script tests passed"