From 44775ae2fe60f7238fbf211a34b27c83710516a1 Mon Sep 17 00:00:00 2001 From: White-Lotus <95460219+mangyan1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:43:39 -0600 Subject: [PATCH 1/3] make aborting safe updates loud instead of silent set -e aborts the safe-update wrapper on any unguarded failing command, but only the update dispatch carries a revert guard: every other phase died silently, leaving the stack half-maintained with no trace of why (a 2026-10-03 run ended this way in an OCI exec error and rewrote nothing). The wrapper now traps those deaths: it logs the failing command and wrapper line, writes aborted-{ps,logs}.txt diagnostics beside the last-good snapshot, and ends the phase correctly - no revert before deployment because the running stack is untouched, revert after it. The rollback database snapshot retries its container exec three times first, since an exec can transiently lose a race against a container being stopped or recreated; a stack whose rollback database cannot be created is never updated. The interactive --update path installs the same abort naming so a bare failure names the command and script line that killed the run. Safe-update wrapper v8; installer v1.68. --- .github/workflows/smoke-test.yml | 4 +-- CHANGELOG.md | 2 ++ setup-npmplus.sh | 54 +++++++++++++++++++++++++++++--- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 59b6c51bb4..a695ff14df 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -363,7 +363,7 @@ jobs: # failure: Docker preserved an empty resolver file in the existing # host-networked container. The downloaded current script must repair # all three before entering the transactional update. - sudo sed -i 's/NPMPLUS_SAFE_UPDATE_WRAPPER_VERSION=7/NPMPLUS_SAFE_UPDATE_WRAPPER_VERSION=3/' /usr/local/bin/npmplus-safe-update + sudo sed -i 's/NPMPLUS_SAFE_UPDATE_WRAPPER_VERSION=8/NPMPLUS_SAFE_UPDATE_WRAPPER_VERSION=3/' /usr/local/bin/npmplus-safe-update sudo chmod 0644 /opt/npmplus/setup-npmplus.sh for _ in $(seq 1 30); do health="$(sudo docker inspect --format '{{.State.Health.Status}}' npmplus)" @@ -434,7 +434,7 @@ jobs: fi rm -f "$update_log" sudo test -x /opt/npmplus/setup-npmplus.sh - sudo grep -qx '# NPMPLUS_SAFE_UPDATE_WRAPPER_VERSION=7' /usr/local/bin/npmplus-safe-update + sudo grep -qx '# NPMPLUS_SAFE_UPDATE_WRAPPER_VERSION=8' /usr/local/bin/npmplus-safe-update sudo test -f /var/backups/npmplus-last-good/override.yaml test "$(stat -c '%u:%g' /opt/anubis-data)" = '1000:1000' sudo test -f /opt/anubis-data/anubis/honeypot.addrs diff --git a/CHANGELOG.md b/CHANGELOG.md index 65af4fb595..313d03378b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to the NPMplus Security Fork are documented here. The fork u ## Unreleased +- Installer v1.68 / safe-update wrapper v8 make every aborting update path loud instead of silent. The wrapper traps unguarded failures: an aborted run logs the failing command and wrapper line, writes `aborted-{ps,logs}.txt` diagnostics beside the last-good snapshot, and ends correctly for its phase — no revert before deployment (the running stack is untouched), revert after it. The rollback database snapshot retries its container exec three times before aborting, because an exec can transiently lose a race against a container being stopped or recreated by another actor; a stack whose rollback database cannot be created is never updated. The interactive `--update` path installs the same abort naming, so a bare failure always reports the command and script line that killed the run. + - Installer v1.67 / safe-update wrapper v7 refuse to deploy an older setup script: a candidate whose `SCRIPT_VERSION` predates the installed script is rejected while the stack is still healthy, since an older script pins older image channels over a database already migrated by newer code and the app crash-loops at startup until the health check reverts. `NPMPLUS_FORCE_DOWNGRADE=true` overrides the refusal. `pin_image` also no longer treats a failed `docker pull` as success — a transient registry error previously re-pinned the local image's stale digest and reported a healthy update on yesterday's image; the pin now fails so the wrapper reverts instead. - Installer v1.66 adds an admin-only **Enable CRS observation** button under CrowdSec → WAF for managed stacks with AppSec enabled. A private, single-action host helper validates the shared observation policy, snapshots configuration, waits for CrowdSec health, and rolls back failures without exposing Docker or arbitrary host commands to the application. Ordinary installer updates install the helper; existing deployments require that update once. diff --git a/setup-npmplus.sh b/setup-npmplus.sh index 008824c51b..2ba3c07727 100644 --- a/setup-npmplus.sh +++ b/setup-npmplus.sh @@ -11,7 +11,7 @@ set -euo pipefail # bump this on every meaningful change - the script compares it against the # copy on github at startup and tells the operator when theirs is stale -SCRIPT_VERSION="1.67" +SCRIPT_VERSION="1.68" DATA_DIR="/opt/npmplus" CROWDSEC_DIR="/opt/crowdsec" @@ -3003,7 +3003,7 @@ if [[ -s "$DATA_DIR/setup-npmplus.sh" ]]; then chmod 700 "$DATA_DIR/setup-npmplus.sh" write_root_file /usr/local/bin/npmplus-safe-update 700 <<'EOF' #!/bin/bash -# NPMPLUS_SAFE_UPDATE_WRAPPER_VERSION=7 +# NPMPLUS_SAFE_UPDATE_WRAPPER_VERSION=8 # monthly npmplus update with a safety net: snapshots the running state, # runs the update, health-checks it, and reverts to the snapshot on failure set -euo pipefail @@ -3019,6 +3019,26 @@ log() { echo "$(date '+%F %T') $*"; } exec 9>/run/lock/npmplus-maintenance.lock flock -n 9 || { log "another NPMplus maintenance job is already running"; exit 1; } +# set -e aborts on any unguarded failing command, but only the update dispatch +# below carries a revert guard; every other phase would otherwise die silently +# with the stack left half-maintained and no trace of why. Catch those deaths, +# leave diagnostics, and end the run the way the phase requires. +update_deployed=false +on_wrapper_error() { + local rc=$? line=${BASH_LINENO[0]} cmd="$BASH_COMMAND" + trap - ERR + log "update ABORTED: $cmd failed with exit $rc (wrapper line $line)" + docker compose -f "$COMPOSE_FILE" ps -a >"$BACKUP/aborted-ps.txt" 2>&1 || true + docker compose -f "$COMPOSE_FILE" logs --no-color --tail 200 >"$BACKUP/aborted-logs.txt" 2>&1 || true + if [[ "$update_deployed" == "true" ]]; then + log "the update was already deployed - reverting" + revert + else + log "the running stack was not updated - no revert needed; diagnostics are in $BACKUP/aborted-{ps,logs}.txt" + fi +} +trap on_wrapper_error ERR + revert() { log "health check FAILED - reverting to the last good state" docker compose -f "$COMPOSE_FILE" ps -a >"$BACKUP/failed-ps.txt" 2>&1 || true @@ -3196,7 +3216,23 @@ fi rm -f "$BACKUP/database.sqlite" if [[ -f /opt/npmplus/npmplus/database.sqlite ]]; then rm -f /opt/npmplus/npmplus/database.pre-update.sqlite - docker exec npmplus node -e "const d=require('better-sqlite3')('/data/npmplus/database.sqlite',{readonly:true});d.backup('/data/npmplus/database.pre-update.sqlite').then(()=>d.close())" + # a docker exec can transiently lose a race against a container being + # stopped or recreated by another actor; retry before treating it as a + # real failure. A stack whose rollback database cannot be created is + # never updated. + db_backup_done=false + for _ in 1 2 3; do + if docker exec npmplus node -e "const d=require('better-sqlite3')('/data/npmplus/database.sqlite',{readonly:true});d.backup('/data/npmplus/database.pre-update.sqlite').then(()=>d.close())"; then + db_backup_done=true + break + fi + log "rollback database snapshot exec failed - retrying" + sleep 5 + done + if [[ "$db_backup_done" != "true" ]]; then + log "cannot snapshot the rollback database - aborting before the update" + exit 1 + fi cp -a /opt/npmplus/npmplus/database.pre-update.sqlite "$BACKUP/database.sqlite" chmod 600 "$BACKUP/database.sqlite" rm -f /opt/npmplus/npmplus/database.pre-update.sqlite @@ -3259,6 +3295,7 @@ if [[ -n "$CANDIDATE" && "$CANDIDATE" != "$SETUP" ]]; then chmod 700 "$SETUP" || revert fi NPMPLUS_SAFE_UPDATE_ACTIVE=true "$SETUP" --update || revert +update_deployed=true # crowdsec hub: refresh the detection signatures (parsers/scenarios/collections); # a container image update alone never touches them and they live outside the image @@ -3812,6 +3849,15 @@ if [[ "${1:-}" == "--update" ]]; then echo "no existing install at $COMPOSE_FILE - run without --update first" >&2 exit 1 fi + # this unattended path aborts on any unguarded failing command; name the + # failing command and line so an aborted update is never silent. When it + # runs as the wrapper's child, the wrapper's revert and failed-*.txt + # diagnostics take over from here. + on_update_abort() { + local rc=$? line=${BASH_LINENO[0]} cmd="$BASH_COMMAND" + printf 'update ABORTED: %s failed with exit %s (script line %s)\n' "$cmd" "$rc" "$line" >&2 + } + trap on_update_abort ERR # RC-era retries could leave the package and our generated configuration in # place without the ownership marker when the binary already existed. Adopt # only installations carrying both the NPMplus Compose service and our config @@ -3865,7 +3911,7 @@ if [[ "${1:-}" == "--update" ]]; then # overwrite themselves in place during a nested tooling refresh. Replace # them before delegation; this also repairs a missing execute bit. if [[ ! -x /usr/local/bin/npmplus-safe-update ]] || \ - ! grep -qx '# NPMPLUS_SAFE_UPDATE_WRAPPER_VERSION=7' /usr/local/bin/npmplus-safe-update; then + ! grep -qx '# NPMPLUS_SAFE_UPDATE_WRAPPER_VERSION=8' /usr/local/bin/npmplus-safe-update; then install_host_tooling fi # Repair the v1.6 root-owned Anubis bind mount before the wrapper checks From af8187922ec950dc217ded73df80f0721c9e407d Mon Sep 17 00:00:00 2001 From: White-Lotus <95460219+mangyan1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:51:27 -0600 Subject: [PATCH 2/3] refuse running an older setup script over a newer install A release script older than the installed one rewrites the safe-update wrapper with its own before the wrapper's candidate guard can run, then pins older image channels over a database already migrated by newer code. The --update path now compares its SCRIPT_VERSION against the installed copy and refuses before any tooling or wrapper refresh runs; NPMPLUS_FORCE_DOWNGRADE=true overrides, and a fresh install without an installed copy proceeds untouched. --- setup-npmplus.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup-npmplus.sh b/setup-npmplus.sh index 2ba3c07727..b52ea23b33 100644 --- a/setup-npmplus.sh +++ b/setup-npmplus.sh @@ -3849,6 +3849,18 @@ if [[ "${1:-}" == "--update" ]]; then echo "no existing install at $COMPOSE_FILE - run without --update first" >&2 exit 1 fi + # A release script older than the installed one would rewrite the + # safe-update wrapper and pin older image channels over a database + # already migrated by newer code; refuse it while the stack is still + # healthy, before any tooling or wrapper refresh runs. + installed_version=$(sed -n 's/^SCRIPT_VERSION="\([^"]*\)".*/\1/p' "$DATA_DIR/setup-npmplus.sh" 2>/dev/null | head -1) || true + if [[ -n "$installed_version" && "$installed_version" != "$SCRIPT_VERSION" ]] && + [[ "$(printf '%s\n%s\n' "$SCRIPT_VERSION" "$installed_version" | sort -V | tail -1)" == "$installed_version" ]] && + [[ "${NPMPLUS_FORCE_DOWNGRADE:-false}" != "true" ]]; then + echo "this setup script v$SCRIPT_VERSION is older than the installed v$installed_version - refusing the downgrade" >&2 + echo "the running stack is untouched; re-download the current script, or set NPMPLUS_FORCE_DOWNGRADE=true to override" >&2 + exit 1 + fi # this unattended path aborts on any unguarded failing command; name the # failing command and line so an aborted update is never silent. When it # runs as the wrapper's child, the wrapper's revert and failed-*.txt From 370380ded63d92ad26d2f159b5f1981b0486b50b Mon Sep 17 00:00:00 2001 From: White-Lotus <95460219+mangyan1@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:23:31 -0600 Subject: [PATCH 3/3] mark the trap-invoked abort handler for shellcheck --- setup-npmplus.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup-npmplus.sh b/setup-npmplus.sh index b52ea23b33..58e9f3b3b2 100644 --- a/setup-npmplus.sh +++ b/setup-npmplus.sh @@ -3865,6 +3865,8 @@ if [[ "${1:-}" == "--update" ]]; then # failing command and line so an aborted update is never silent. When it # runs as the wrapper's child, the wrapper's revert and failed-*.txt # diagnostics take over from here. + # Invoked by the ERR trap below. + # shellcheck disable=SC2329 on_update_abort() { local rc=$? line=${BASH_LINENO[0]} cmd="$BASH_COMMAND" printf 'update ABORTED: %s failed with exit %s (script line %s)\n' "$cmd" "$rc" "$line" >&2