From 079a96b0431bdbf9fc0d3b47257cd9358b9488aa Mon Sep 17 00:00:00 2001 From: Openclaw Date: Mon, 14 Sep 2026 19:06:11 +0200 Subject: [PATCH 1/4] build: run the Docker runtime on Alpine 3.24.1 with Mosquitto 2.1 compat Move the runtime stage of Dockerfile and Dockerfile.go from the EOL alpine:3.20 to alpine:3.24.1, pinned by digest. The Go builder (golang:1.27.1-alpine3.24, digest-pinned) is unchanged, and the Go binaries are byte-identical to the Alpine 3.20 image per architecture. Alpine 3.24 ships Mosquitto 2.1.2, which changed two behaviours the in-container broker relied on: - allow_duplicate_messages now defaults to true. The ingestor can hold overlapping subscriptions (meshcore/+/+/packets + meshcore/#), so each packet was delivered twice and packetsTotal, tx_dupes, observer upserts and observation rows doubled. docker/mosquitto.conf sets it back to false. Mosquitto 2.0.18 and 2.1.2 both honour it and both log a deprecation warning; upstream plans to remove it in Mosquitto 3.0. - PUID/PGID are honoured for privilege dropping. entrypoint-go.sh exports /app/data/.env, so a PUID there made the broker run as that uid and fail to write /var/lib/mosquitto/mosquitto.db. The supervisord programs that start the broker now strip PUID/PGID with `env -u` for mosquitto only; every other process keeps its environment and user. scripts/test-docker-mosquitto-compat.sh is a behavioural regression test for both. It runs a built image with synthetic data and --network none, publishes to the in-container broker and checks deliveries, ingestor counters, the broker's uid/gid and persistence across a broker restart. It fails on the unfixed Alpine 3.24.1 image and passes with this change. It is not registered in CI. Dockerfile.go still fails at `go mod download` because of missing COPY internal/ lines; that pre-existing issue is out of scope here. Co-Authored-By: Claude Opus 5 --- Dockerfile | 2 +- Dockerfile.go | 2 +- docker/mosquitto.conf | 7 + docker/supervisord-go-no-caddy.conf | 5 +- docker/supervisord-go.conf | 5 +- scripts/test-docker-mosquitto-compat.sh | 192 ++++++++++++++++++++++++ 6 files changed, 209 insertions(+), 4 deletions(-) create mode 100755 scripts/test-docker-mosquitto-compat.sh diff --git a/Dockerfile b/Dockerfile index 8d39bb7e0..dc8a2165a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,7 +57,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -ldflags="-s -w" -o /corescope-decrypt . # Runtime image -FROM alpine:3.20 +FROM alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget diff --git a/Dockerfile.go b/Dockerfile.go index 5cd8e5cac..52b0012aa 100644 --- a/Dockerfile.go +++ b/Dockerfile.go @@ -21,7 +21,7 @@ COPY cmd/ingestor/ ./ RUN go build -o /corescope-ingestor . # Runtime image -FROM alpine:3.20 +FROM alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget diff --git a/docker/mosquitto.conf b/docker/mosquitto.conf index 86f274014..1b8d95010 100644 --- a/docker/mosquitto.conf +++ b/docker/mosquitto.conf @@ -4,6 +4,13 @@ allow_anonymous true persistence true persistence_location /var/lib/mosquitto/ +# Mosquitto >= 2.1 defaults allow_duplicate_messages to true: a client with +# overlapping subscriptions (the ingestor's meshcore/+/+/packets + meshcore/#) +# then receives every message once per matching subscription. Keep one +# delivery per client. Deprecated upstream (removal planned for Mosquitto 3.0); +# the broker logs a deprecation warning at startup. +allow_duplicate_messages false + # Logging log_dest stdout log_type warning diff --git a/docker/supervisord-go-no-caddy.conf b/docker/supervisord-go-no-caddy.conf index 3ca0af00f..6d4e5fe99 100644 --- a/docker/supervisord-go-no-caddy.conf +++ b/docker/supervisord-go-no-caddy.conf @@ -6,7 +6,10 @@ logfile_maxbytes=0 pidfile=/var/run/supervisord.pid [program:mosquitto] -command=/usr/sbin/mosquitto -c /etc/mosquitto/mosquitto.conf +; Mosquitto >= 2.1 drops privileges to PUID/PGID when they are set, and +; entrypoint-go.sh exports /app/data/.env. Strip them for the broker only so it +; keeps running as the mosquitto user that owns /var/lib/mosquitto. +command=/usr/bin/env -u PUID -u PGID /usr/sbin/mosquitto -c /etc/mosquitto/mosquitto.conf autostart=true autorestart=true stdout_logfile=/dev/stdout diff --git a/docker/supervisord-go.conf b/docker/supervisord-go.conf index 315b10ab6..962743ec8 100644 --- a/docker/supervisord-go.conf +++ b/docker/supervisord-go.conf @@ -6,7 +6,10 @@ logfile_maxbytes=0 pidfile=/var/run/supervisord.pid [program:mosquitto] -command=/usr/sbin/mosquitto -c /etc/mosquitto/mosquitto.conf +; Mosquitto >= 2.1 drops privileges to PUID/PGID when they are set, and +; entrypoint-go.sh exports /app/data/.env. Strip them for the broker only so it +; keeps running as the mosquitto user that owns /var/lib/mosquitto. +command=/usr/bin/env -u PUID -u PGID /usr/sbin/mosquitto -c /etc/mosquitto/mosquitto.conf autostart=true autorestart=true stdout_logfile=/dev/stdout diff --git a/scripts/test-docker-mosquitto-compat.sh b/scripts/test-docker-mosquitto-compat.sh new file mode 100755 index 000000000..cf13a41ec --- /dev/null +++ b/scripts/test-docker-mosquitto-compat.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# scripts/test-docker-mosquitto-compat.sh +# +# Runtime regression test for the in-container Mosquitto broker of a built +# CoreScope image (Dockerfile / Dockerfile.go runtime stage). It guards two +# Mosquitto 2.1 behaviour changes that arrived with the Alpine 3.24 runtime: +# +# 1. allow_duplicate_messages defaults to true in 2.1. The ingestor can hold +# overlapping subscriptions (meshcore/+/+/packets + meshcore/#), so every +# packet was delivered twice: packetsTotal, tx_dupes, observer upserts and +# observation rows doubled. +# 2. Mosquitto 2.1 drops privileges to PUID/PGID when they are set. +# entrypoint-go.sh exports /app/data/.env, so a PUID there made the broker +# run as that uid and fail to write /var/lib/mosquitto/mosquitto.db. +# +# Behavioural, not a config grep: it starts the image with synthetic data and +# --network none, publishes synthetic messages to the in-container broker and +# checks broker deliveries, ingestor counters, the broker's uid/gid and +# persistence across a broker restart. One container runs at a time. +# +# usage: scripts/test-docker-mosquitto-compat.sh [platform] +# e.g. scripts/test-docker-mosquitto-compat.sh corescope-go:latest linux/arm64 +# Containers are created with --rm, so they and their anonymous volumes are +# removed when the test stops them. LOG_DIR= keeps each container's logs; +# WAIT_S (default 240) bounds every wait (raise it for emulated platforms). +set -uo pipefail + +IMAGE=${1:?usage: $0 [platform]} +PLATFORM=${2:-} +WAIT_S=${WAIT_S:-240} +LOG_DIR=${LOG_DIR:-} +RUN_ID="mqttcompat-$$" +OBSERVER=00000000000000000000000000000000000000000000000000000000c0ffee01 +TOPIC="meshcore/TST/$OBSERVER/packets" +PAYLOAD='{"raw":"0A00D69FD7A5A7475DB07337749AE61FA53A4788E976","SNR":5.0,"RSSI":-100.0,"origin":"mqtt-compat-test"}' +RETAINED_TOPIC="corescope-compat/retained" +RETAINED_MSG="retained-$RUN_ID" +CHECKS=0 +FAILS=0 +CONTAINERS="" +TMP=$(mktemp -d) + +stop_container() { # name: keep its logs (LOG_DIR), stop it (--rm removes it) + local c=$1 x rest="" + if [ -n "$LOG_DIR" ]; then + mkdir -p "$LOG_DIR" + docker logs -t "$c" > "$LOG_DIR/$c.log" 2>&1 + fi + docker stop -t 30 "$c" > /dev/null 2>&1 + for x in $CONTAINERS; do [ "$x" = "$c" ] || rest="$rest $x"; done + CONTAINERS=$rest +} + +cleanup() { + local c + for c in $CONTAINERS; do stop_container "$c"; done + rm -rf "$TMP" +} +trap cleanup EXIT + +check() { # description expected actual + CHECKS=$((CHECKS + 1)) + if [ "$2" = "$3" ]; then + echo "ok - $1 ($3)" + else + echo "FAIL - $1: expected '$2', got '$3'" + FAILS=$((FAILS + 1)) + fi +} + +wait_for() { # timeout_s command... + local t=$1 i + shift + for ((i = 0; i < t; i++)); do + "$@" > /dev/null 2>&1 && return 0 + sleep 1 + done + return 1 +} + +dx() { docker exec "$@"; } + +start_container() { # name [file copied into /app/data ...] + local name=$1 f + shift + local plat=() + [ -n "$PLATFORM" ] && plat=(--platform "$PLATFORM") + docker create --rm --name "$name" --network none ${plat[@]+"${plat[@]}"} "$IMAGE" > /dev/null || return 1 + CONTAINERS="$CONTAINERS $name" + for f in "$@"; do + docker cp "$f" "$name:/app/data/" > /dev/null || return 1 + done + docker start "$name" > /dev/null +} + +json_int() { grep -oE "\"$1\":[0-9]+" | head -1 | cut -d: -f2; } +server_up() { dx "$1" wget -qO /dev/null http://localhost:3000/api/mqtt/status; } +subscribed_topics() { docker logs "$1" 2>&1 | grep -oE 'MQTT \[[a-z]+\] subscribed to [^ ]+' | sort -u | wc -l | tr -d ' '; } +subscribed_at_least() { [ "$(subscribed_topics "$1")" -ge "$2" ]; } +stats_file_int() { dx "$1" cat /tmp/corescope-ingestor-stats.json 2> /dev/null | json_int "$2"; } +obs_at_least() { [ "$(stats_file_int "$1" obs_inserted)" -ge "$2" ] 2> /dev/null; } +no_subscriber_running() { ! dx "$1" pidof mosquitto_sub; } +ids_of() { dx "$1" awk '/^Uid:/{u=$2} /^Gid:/{g=$2} END{print u ":" g}' "/proc/$2/status"; } +env_has() { dx "$1" sh -c "tr '\\0' '\\n' < /proc/$2/environ | grep -qx '$3'"; } +broker_restarted() { local p; p=$(dx "$1" pidof mosquitto) && [ -n "$p" ] && [ "$p" != "$2" ]; } +retained_present() { [ "$(dx "$1" mosquitto_sub -h localhost -t "$RETAINED_TOPIC" -C 1 -W 3 2> /dev/null)" = "$RETAINED_MSG" ]; } + +# Broker identity + persistence: publish a retained message, SIGTERM the broker +# (it saves its database on exit), let supervisord restart it, then require the +# retained message, the database owner and a clean log. +check_broker_persistence() { # container label + local c=$1 label=$2 want pid newpid + want=$(dx "$c" sh -c 'echo "$(id -u mosquitto):$(id -g mosquitto)"') + pid=$(dx "$c" pidof mosquitto) + check "$label: broker runs as the mosquitto user" "$want" "$(ids_of "$c" "$pid")" + dx "$c" mosquitto_pub -h localhost -q 1 -r -t "$RETAINED_TOPIC" -m "$RETAINED_MSG" + dx "$c" kill -TERM "$pid" + if ! wait_for "$WAIT_S" broker_restarted "$c" "$pid"; then + check "$label: supervisord restarted the broker" "yes" "no" + return + fi + newpid=$(dx "$c" pidof mosquitto) + check "$label: restarted broker runs as the mosquitto user" "$want" "$(ids_of "$c" "$newpid")" + wait_for 60 retained_present "$c" + check "$label: retained message survives a broker restart" "$RETAINED_MSG" \ + "$(dx "$c" mosquitto_sub -h localhost -t "$RETAINED_TOPIC" -C 1 -W 5 2> /dev/null)" + check "$label: persistence database owned by the broker user" "$want" \ + "$(dx "$c" stat -c '%u:%g' /var/lib/mosquitto/mosquitto.db 2> /dev/null)" + check "$label: no persistence write errors logged" "0" \ + "$(docker logs "$c" 2>&1 | grep -cE 'Error saving|Permission denied')" +} + +echo "# image: $IMAGE platform: ${PLATFORM:-default}" + +# --------------------------------------------------------------------------- +echo "# case 1: overlapping subscriptions, no PUID/PGID" +C1="corescope-$RUN_ID-overlap" +mkdir -p "$TMP/overlap" +cat > "$TMP/overlap/config.json" << 'EOF' +{"dbPath": "/app/data/meshcore.db", + "mqttSources": [{"name": "local", "broker": "mqtt://localhost:1883", + "topics": ["meshcore/+/+/packets", "meshcore/#"], "connectTimeoutSec": 30}]} +EOF +start_container "$C1" "$TMP/overlap/config.json" || { echo "FAIL - could not start $C1"; exit 1; } +wait_for "$WAIT_S" server_up "$C1" || check "case 1: server ready" "yes" "no" +wait_for "$WAIT_S" subscribed_at_least "$C1" 2 || check "case 1: ingestor subscribed to both topics" "2" "$(subscribed_topics "$C1")" +dx -d "$C1" sh -c "mosquitto_sub -h localhost -v -t 'meshcore/+/+/packets' -t 'meshcore/#' -W 20 > /tmp/compat-overlap.txt 2>&1" +dx -d "$C1" sh -c "mosquitto_sub -h localhost -v -t 'meshcore/#' -W 20 > /tmp/compat-single.txt 2>&1" +sleep 5 +dx "$C1" mosquitto_pub -h localhost -q 0 -t "$TOPIC" -m "$PAYLOAD" +wait_for "$WAIT_S" obs_at_least "$C1" 1 || check "case 1: ingestor stored the packet" "yes" "no" +wait_for 60 no_subscriber_running "$C1" +sleep 12 # > /api/stats cache TTL (10s); the ingestor stats file is rewritten every second +check "case 1: deliveries to one client with overlapping subscriptions" "1" "$(dx "$C1" grep -c "^$TOPIC " /tmp/compat-overlap.txt)" +check "case 1: deliveries to a client with a single subscription" "1" "$(dx "$C1" grep -c "^$TOPIC " /tmp/compat-single.txt)" +check "case 1: /api/mqtt/status packetsTotal" "1" "$(dx "$C1" wget -qO- http://localhost:3000/api/mqtt/status | json_int packetsTotal)" +check "case 1: ingestor tx_inserted" "1" "$(stats_file_int "$C1" tx_inserted)" +check "case 1: ingestor tx_dupes" "0" "$(stats_file_int "$C1" tx_dupes)" +check "case 1: ingestor obs_inserted" "1" "$(stats_file_int "$C1" obs_inserted)" +check "case 1: ingestor observer_upserts" "1" "$(stats_file_int "$C1" observer_upserts)" +STATS=$(dx "$C1" wget -qO- http://localhost:3000/api/stats) +check "case 1: /api/stats totalTransmissions" "1" "$(echo "$STATS" | json_int totalTransmissions)" +check "case 1: /api/stats totalObservations" "1" "$(echo "$STATS" | json_int totalObservations)" +check_broker_persistence "$C1" "case 1" +stop_container "$C1" + +# --------------------------------------------------------------------------- +echo "# case 2: PUID=1000 PGID=1000 in /app/data/.env (exported by the entrypoint)" +C2="corescope-$RUN_ID-puid" +mkdir -p "$TMP/puid" +printf 'PUID=1000\nPGID=1000\n' > "$TMP/puid/.env" +start_container "$C2" "$TMP/puid/.env" || { echo "FAIL - could not start $C2"; exit 1; } +wait_for "$WAIT_S" server_up "$C2" || check "case 2: server ready" "yes" "no" +wait_for "$WAIT_S" subscribed_at_least "$C2" 1 || check "case 2: ingestor subscribed" "1" "$(subscribed_topics "$C2")" +check "case 2: supervisord (pid 1) keeps PUID/PGID" "yes" \ + "$(env_has "$C2" 1 PUID=1000 && env_has "$C2" 1 PGID=1000 && echo yes || echo no)" +check "case 2: supervisord still runs as root" "0:0" "$(ids_of "$C2" 1)" +for p in corescope-ingestor corescope-server caddy; do + pid=$(dx "$C2" pidof "$p") + check "case 2: $p keeps PUID/PGID" "yes" \ + "$(env_has "$C2" "$pid" PUID=1000 && env_has "$C2" "$pid" PGID=1000 && echo yes || echo no)" + check "case 2: $p still runs as root" "0:0" "$(ids_of "$C2" "$pid")" +done +dx "$C2" mosquitto_pub -h localhost -q 0 -t "$TOPIC" -m "$PAYLOAD" +wait_for "$WAIT_S" obs_at_least "$C2" 1 +check "case 2: ingestor stored the packet via the broker" "1" "$(stats_file_int "$C2" tx_inserted)" +check_broker_persistence "$C2" "case 2" +stop_container "$C2" + +echo "# $CHECKS checks, $FAILS failed" +if [ "$FAILS" -eq 0 ]; then echo "RESULT: PASS"; else echo "RESULT: FAIL"; fi +[ "$FAILS" -eq 0 ] From 3c1b7ca1be957edc9e568c03a83fd7f6ce81d25f Mon Sep 17 00:00:00 2001 From: Openclaw Date: Tue, 15 Sep 2026 08:51:21 +0200 Subject: [PATCH 2/4] test(docker): harden the Mosquitto compat regression test Follow-up to the independent review of scripts/test-docker-mosquitto-compat.sh. Only the test changes; the image, broker config and supervisord files are untouched and the existing 28 functional checks and their expected values stay as they were. - Cleanup: containers get a unique com.corescope.test.run= label and are registered right after `docker create`. On pass, failure, start errors and SIGINT/SIGTERM/SIGHUP the test keeps their logs, removes only containers carrying its own run label together with their anonymous volumes (docker rm -f -v, with a checked fallback) and verifies both are gone. A failing run keeps its non-zero status; a cleanup failure turns a passing run into exit 3. - No registry contact: the image reference must exist locally, is resolved once to its image ID, and every container is created from that ID with --pull never. A missing image fails before any container is created. - Wall-clock limits: TOTAL_S (whole run, with a cleanup reserve), WAIT_S (each wait) and EXEC_S (each docker command) replace iteration-counted loops; wget uses -T. SIGKILL still cannot run the cleanup, so the header keeps recommending an outer timeout. - Completion: every former fire-and-forget wait is now a precondition that stops the run when it does not hold. Exit 0 requires all 28 checks and all 15 preconditions to have run and passed; the image ID and the Mosquitto version are printed. The test still covers only the default supervisord start path and says nothing about Caddy start-up crashes. Co-Authored-By: Claude Opus 5 --- scripts/test-docker-mosquitto-compat.sh | 436 ++++++++++++++++++++---- 1 file changed, 364 insertions(+), 72 deletions(-) diff --git a/scripts/test-docker-mosquitto-compat.sh b/scripts/test-docker-mosquitto-compat.sh index cf13a41ec..b80c502ca 100755 --- a/scripts/test-docker-mosquitto-compat.sh +++ b/scripts/test-docker-mosquitto-compat.sh @@ -7,8 +7,8 @@ # # 1. allow_duplicate_messages defaults to true in 2.1. The ingestor can hold # overlapping subscriptions (meshcore/+/+/packets + meshcore/#), so every -# packet was delivered twice: packetsTotal, tx_dupes, observer upserts and -# observation rows doubled. +# packet was delivered twice: packetsTotal, observation rows and observer +# upserts doubled and tx_dupes went from 0 to 1. # 2. Mosquitto 2.1 drops privileges to PUID/PGID when they are set. # entrypoint-go.sh exports /app/data/.env, so a PUID there made the broker # run as that uid and fail to write /var/lib/mosquitto/mosquitto.db. @@ -17,48 +17,164 @@ # --network none, publishes synthetic messages to the in-container broker and # checks broker deliveries, ingestor counters, the broker's uid/gid and # persistence across a broker restart. One container runs at a time. +# Scope: the default supervisord start path only. It does not exercise +# DISABLE_CADDY, does not watch for unexpected restarts of other supervisord +# programs, and a PASS is no evidence that Caddy starts without crashing. # # usage: scripts/test-docker-mosquitto-compat.sh [platform] # e.g. scripts/test-docker-mosquitto-compat.sh corescope-go:latest linux/arm64 -# Containers are created with --rm, so they and their anonymous volumes are -# removed when the test stops them. LOG_DIR= keeps each container's logs; -# WAIT_S (default 240) bounds every wait (raise it for emulated platforms). +# +# The image must already exist locally. It is never pulled: the reference is +# resolved to its image ID first and every container is created from that ID +# with --pull never. Containers carry the labels com.corescope.test=mosquitto-compat +# and com.corescope.test.run=. On normal exit, failures and +# SIGINT/SIGTERM/SIGHUP the test keeps each container's logs, removes exactly +# its own labelled containers and their anonymous volumes, and verifies they +# are gone. SIGKILL cannot run that cleanup, so still run the test under an +# outer timeout; leftovers of a killed run can be listed with +# docker ps -a --filter label=com.corescope.test.run= +# +# Environment (wall-clock seconds): +# TOTAL_S budget for the whole run, including a 45s cleanup reserve (default 900) +# WAIT_S limit for each readiness/progress wait (default 240) +# EXEC_S limit for each single docker command (default 30) +# LOG_DIR where container logs are kept +# (default ${TMPDIR:-/tmp}/corescope-mqtt-compat-) +# +# Exit status: 0 all 28 checks and all 15 preconditions ran and passed; +# 1 a check or precondition failed, the budget ran out or the run was incomplete; +# 2 setup error (bad arguments, no docker, image not local, platform mismatch); +# 3 all checks passed but cleanup failed; 128+N stopped by signal N. set -uo pipefail -IMAGE=${1:?usage: $0 [platform]} +IMAGE=${1:-} PLATFORM=${2:-} +TOTAL_S=${TOTAL_S:-900} WAIT_S=${WAIT_S:-240} -LOG_DIR=${LOG_DIR:-} -RUN_ID="mqttcompat-$$" +EXEC_S=${EXEC_S:-30} +CLEANUP_RESERVE_S=45 +EXPECTED_CHECKS=28 +EXPECTED_PRECONDITIONS=15 +RUN_ID="mqttcompat-$(date +%Y%m%d%H%M%S)-$$-$RANDOM" +LABEL_TEST="com.corescope.test" +LABEL_RUN="com.corescope.test.run" +LOG_DIR=${LOG_DIR:-${TMPDIR:-/tmp}/corescope-mqtt-compat-$RUN_ID} OBSERVER=00000000000000000000000000000000000000000000000000000000c0ffee01 TOPIC="meshcore/TST/$OBSERVER/packets" PAYLOAD='{"raw":"0A00D69FD7A5A7475DB07337749AE61FA53A4788E976","SNR":5.0,"RSSI":-100.0,"origin":"mqtt-compat-test"}' RETAINED_TOPIC="corescope-compat/retained" RETAINED_MSG="retained-$RUN_ID" + CHECKS=0 FAILS=0 -CONTAINERS="" -TMP=$(mktemp -d) +PRECONDITIONS=0 +PRECONDITION_FAILS=0 +COMPLETED=0 +BUDGET_EXHAUSTED=0 +CLEANUP_FAILED=0 +RESULT_PRINTED=0 +IN_CLEANUP=0 +OWNED="" # IDs of the containers this run created +OWNED_VOLUMES="" # anonymous volumes those containers were created with +LAST_ID="" +BG_PID="" +TMP="" + +setup_error() { + echo "RESULT: ERROR - $1" + RESULT_PRINTED=1 + exit 2 +} + +case "$TOTAL_S$WAIT_S$EXEC_S" in *[!0-9]*) setup_error "TOTAL_S, WAIT_S and EXEC_S must be whole seconds" ;; esac +[ -n "$IMAGE" ] || setup_error "usage: $0 [platform]" +[ "$TOTAL_S" -gt $((CLEANUP_RESERVE_S + 10)) ] || setup_error "TOTAL_S must be greater than $((CLEANUP_RESERVE_S + 10))" +DEADLINE=$((SECONDS + TOTAL_S - CLEANUP_RESERVE_S)) +CLEANUP_DEADLINE=$((SECONDS + TOTAL_S)) -stop_container() { # name: keep its logs (LOG_DIR), stop it (--rm removes it) - local c=$1 x rest="" - if [ -n "$LOG_DIR" ]; then - mkdir -p "$LOG_DIR" - docker logs -t "$c" > "$LOG_DIR/$c.log" 2>&1 +# --------------------------------------------------------------------------- +# Wall-clock bounded execution + +# run_bounded : run an external command with a wall-clock +# limit, capped by what is left of the run's budget. Returns 124 on timeout. +run_bounded() { + local limit=$1 left end pid rc + shift + if [ "$IN_CLEANUP" = 1 ]; then left=$((CLEANUP_DEADLINE - SECONDS)); else left=$((DEADLINE - SECONDS)); fi + [ "$left" -ge "$limit" ] || limit=$left + [ "$limit" -gt 0 ] || return 124 + "$@" & + pid=$! + BG_PID=$pid + end=$((SECONDS + limit)) + while kill -0 "$pid" 2> /dev/null; do + if [ "$SECONDS" -ge "$end" ]; then + kill -TERM "$pid" 2> /dev/null + sleep 1 + kill -KILL "$pid" 2> /dev/null + wait "$pid" 2> /dev/null + BG_PID="" + return 124 + fi + sleep 0.2 + done + wait "$pid" + rc=$? + BG_PID="" + return "$rc" +} + +pause() { # : interruptible sleep, capped by the run's budget + local s=$1 left=$((DEADLINE - SECONDS)) + [ "$left" -ge "$s" ] || s=$left + [ "$s" -gt 0 ] || return 0 + sleep "$s" & + BG_PID=$! + wait "$BG_PID" + BG_PID="" +} + +wait_until() { # : poll until the predicate succeeds; 1 on timeout + local end=$((SECONDS + $1)) + shift + while :; do + "$@" > /dev/null 2>&1 && return 0 + if [ "$SECONDS" -ge "$end" ] || [ "$SECONDS" -ge "$DEADLINE" ]; then return 1; fi + pause 1 + done +} + +# --------------------------------------------------------------------------- +# Results + +finish() { + local met=$((PRECONDITIONS - PRECONDITION_FAILS)) + echo "# checks: $CHECKS of $EXPECTED_CHECKS run, $FAILS failed; preconditions: $met of $EXPECTED_PRECONDITIONS met" + RESULT_PRINTED=1 + if [ "$FAILS" -ne 0 ] || [ "$PRECONDITION_FAILS" -ne 0 ] || [ "$BUDGET_EXHAUSTED" -ne 0 ]; then + if [ "$COMPLETED" = 1 ]; then echo "RESULT: FAIL"; else echo "RESULT: FAIL (run incomplete)"; fi + exit 1 + fi + if [ "$COMPLETED" != 1 ] || [ "$CHECKS" -ne "$EXPECTED_CHECKS" ] || [ "$met" -ne "$EXPECTED_PRECONDITIONS" ]; then + echo "RESULT: INCOMPLETE" + exit 1 + fi + if [ "$CLEANUP_FAILED" -ne 0 ]; then + echo "RESULT: FAIL (all checks passed, cleanup failed)" + exit 3 fi - docker stop -t 30 "$c" > /dev/null 2>&1 - for x in $CONTAINERS; do [ "$x" = "$c" ] || rest="$rest $x"; done - CONTAINERS=$rest + echo "RESULT: PASS" + exit 0 } -cleanup() { - local c - for c in $CONTAINERS; do stop_container "$c"; done - rm -rf "$TMP" +budget_exhausted() { + echo "FAIL - time budget exhausted (TOTAL_S=${TOTAL_S}s incl. ${CLEANUP_RESERVE_S}s cleanup reserve)" + BUDGET_EXHAUSTED=1 + finish } -trap cleanup EXIT check() { # description expected actual + [ "$SECONDS" -lt "$DEADLINE" ] || budget_exhausted CHECKS=$((CHECKS + 1)) if [ "$2" = "$3" ]; then echo "ok - $1 ($3)" @@ -68,110 +184,287 @@ check() { # description expected actual fi } -wait_for() { # timeout_s command... - local t=$1 i +precondition_failed() { # description detail + PRECONDITION_FAILS=$((PRECONDITION_FAILS + 1)) + echo "FAIL - [precondition] $1: $2" + [ "$SECONDS" -lt "$DEADLINE" ] || budget_exhausted + finish +} + +require() { # description limit predicate...: poll the predicate, stop the run if it never holds + local desc=$1 limit=$2 + shift 2 + PRECONDITIONS=$((PRECONDITIONS + 1)) + if wait_until "$limit" "$@"; then + echo "ok - [precondition] $desc" + elif [ "$SECONDS" -ge "$DEADLINE" ]; then + precondition_failed "$desc" "stopped when the time budget ran out" + else + precondition_failed "$desc" "not met within ${limit}s" + fi +} + +require_now() { # description command...: one attempt, stop the run if it fails + local desc=$1 shift - for ((i = 0; i < t; i++)); do - "$@" > /dev/null 2>&1 && return 0 - sleep 1 - done - return 1 + PRECONDITIONS=$((PRECONDITIONS + 1)) + if "$@"; then + echo "ok - [precondition] $desc" + else + precondition_failed "$desc" "failed" + fi } -dx() { docker exec "$@"; } +# --------------------------------------------------------------------------- +# Containers owned by this run + +volumes_of() { run_bounded "$EXEC_S" docker container inspect -f '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}} {{end}}{{end}}' "$1"; } -start_container() { # name [file copied into /app/data ...] - local name=$1 f +start_container() { # name [file copied into /app/data ...]; sets LAST_ID + local name=$1 id f shift local plat=() [ -n "$PLATFORM" ] && plat=(--platform "$PLATFORM") - docker create --rm --name "$name" --network none ${plat[@]+"${plat[@]}"} "$IMAGE" > /dev/null || return 1 - CONTAINERS="$CONTAINERS $name" + LAST_ID="" + id=$(run_bounded 60 docker create --pull never --name "$name" \ + --label "$LABEL_TEST=mosquitto-compat" --label "$LABEL_RUN=$RUN_ID" \ + --network none ${plat[@]+"${plat[@]}"} "$IMAGE_ID") + if [ -z "$id" ]; then + echo "# could not create $name from $IMAGE_ID" + return 1 + fi + OWNED="$OWNED $id" + LAST_ID=$id + OWNED_VOLUMES="$OWNED_VOLUMES $(volumes_of "$id")" + if [ "$(run_bounded "$EXEC_S" docker container inspect -f '{{.Image}}' "$id")" != "$IMAGE_ID" ]; then + echo "# $name was not created from $IMAGE_ID" + return 1 + fi for f in "$@"; do - docker cp "$f" "$name:/app/data/" > /dev/null || return 1 + if ! run_bounded "$EXEC_S" docker cp "$f" "$id:/app/data/" > /dev/null; then + echo "# could not copy $(basename "$f") into $name:/app/data/" + return 1 + fi done - docker start "$name" > /dev/null + if ! run_bounded 60 docker start "$id" > /dev/null; then + echo "# could not start $name" + return 1 + fi +} + +remove_volume() { # anonymous volume created with one of our containers + local v=$1 rc users labels + run_bounded "$EXEC_S" docker volume inspect "$v" > /dev/null 2>&1 + rc=$? + [ "$rc" -ne 124 ] || { echo "CLEANUP FAIL - timed out inspecting volume $v"; return 1; } + [ "$rc" -eq 0 ] || return 0 + labels=$(run_bounded "$EXEC_S" docker volume inspect -f '{{json .Labels}}' "$v") + case "$labels" in + *'"com.docker.volume.anonymous"'*) ;; + *) echo "CLEANUP SKIP - volume $v is not marked anonymous; left untouched"; return 1 ;; + esac + users=$(run_bounded "$EXEC_S" docker ps -a -q --filter "volume=$v") + if [ -n "$users" ]; then + echo "CLEANUP SKIP - volume $v is still used by $users; left untouched" + return 1 + fi + run_bounded "$EXEC_S" docker volume rm "$v" > /dev/null 2>&1 + run_bounded "$EXEC_S" docker volume inspect "$v" > /dev/null 2>&1 + [ $? -eq 1 ] || { echo "CLEANUP FAIL - volume $v still exists or could not be checked"; return 1; } } +remove_container() { # container id: verify the run label, keep logs, remove it and its anonymous volumes + local id=$1 rc label name vols v + run_bounded "$EXEC_S" docker container inspect "$id" > /dev/null 2>&1 + rc=$? + [ "$rc" -ne 124 ] || { echo "CLEANUP FAIL - timed out inspecting container $id"; return 1; } + [ "$rc" -eq 0 ] || return 0 + label=$(run_bounded "$EXEC_S" docker container inspect -f "{{index .Config.Labels \"$LABEL_RUN\"}}" "$id") + if [ "$label" != "$RUN_ID" ]; then + echo "CLEANUP SKIP - container $id is not labelled $LABEL_RUN=$RUN_ID; left untouched" + return 1 + fi + name=$(run_bounded "$EXEC_S" docker container inspect -f '{{.Name}}' "$id" | tr -d '/') + vols=$(volumes_of "$id") + if mkdir -p "$LOG_DIR" 2> /dev/null; then + run_bounded "$EXEC_S" docker logs -t "$id" > "$LOG_DIR/${name:-$id}.log" 2>&1 + fi + run_bounded 20 docker stop -t 10 "$id" > /dev/null 2>&1 + run_bounded "$EXEC_S" docker rm -f -v "$id" > /dev/null 2>&1 + run_bounded "$EXEC_S" docker container inspect "$id" > /dev/null 2>&1 + [ $? -eq 1 ] || { echo "CLEANUP FAIL - container ${name:-$id} still exists or could not be checked"; return 1; } + for v in $vols; do + remove_volume "$v" || return 1 + done + echo "# removed container ${name:-$id} and its anonymous volumes:" $vols +} + +retire_container() { # container id: remove it now and forget it + local x rest="" + if remove_container "$1"; then + for x in $OWNED; do [ "$x" = "$1" ] || rest="$rest $x"; done + OWNED=$rest + else + CLEANUP_FAILED=1 + fi +} + +cleanup() { + local ok=0 id v listed + listed=$(run_bounded "$EXEC_S" docker ps -a -q --no-trunc --filter "label=$LABEL_RUN=$RUN_ID") + for id in $(printf '%s\n' $OWNED $listed | sort -u); do + remove_container "$id" || ok=1 + done + for v in $(printf '%s\n' $OWNED_VOLUMES | sort -u); do + remove_volume "$v" || ok=1 + done + [ "$CLEANUP_FAILED" = 0 ] || ok=1 + return "$ok" +} + +on_signal() { # number name + echo "# received SIG$2; stopping and cleaning up" + exit $((128 + $1)) +} + +on_exit() { + local rc=$? + trap '' INT TERM HUP + IN_CLEANUP=1 + CLEANUP_DEADLINE=$((SECONDS + CLEANUP_RESERVE_S)) + if [ -n "$BG_PID" ]; then + kill -TERM "$BG_PID" 2> /dev/null + wait "$BG_PID" 2> /dev/null + fi + if [ "$RESULT_PRINTED" != 1 ]; then + echo "# checks: $CHECKS of $EXPECTED_CHECKS run, $FAILS failed; preconditions: $((PRECONDITIONS - PRECONDITION_FAILS)) of $EXPECTED_PRECONDITIONS met" + echo "RESULT: INCOMPLETE" + [ "$rc" -ne 0 ] || rc=1 + fi + if ! cleanup; then + echo "CLEANUP FAIL - check for leftovers labelled $LABEL_RUN=$RUN_ID" + [ "$rc" -ne 0 ] || rc=3 + fi + [ -z "$TMP" ] || rm -rf "$TMP" + exit "$rc" +} + +# --------------------------------------------------------------------------- +# Probes inside a running container + +dx() { run_bounded "$EXEC_S" docker exec "$@"; } +dlogs() { run_bounded "$EXEC_S" docker logs "$@"; } json_int() { grep -oE "\"$1\":[0-9]+" | head -1 | cut -d: -f2; } -server_up() { dx "$1" wget -qO /dev/null http://localhost:3000/api/mqtt/status; } -subscribed_topics() { docker logs "$1" 2>&1 | grep -oE 'MQTT \[[a-z]+\] subscribed to [^ ]+' | sort -u | wc -l | tr -d ' '; } +server_up() { dx "$1" wget -T 5 -qO /dev/null http://localhost:3000/api/mqtt/status; } +subscribed_topics() { dlogs "$1" 2>&1 | grep -oE 'MQTT \[[a-z]+\] subscribed to [^ ]+' | sort -u | wc -l | tr -d ' '; } subscribed_at_least() { [ "$(subscribed_topics "$1")" -ge "$2" ]; } stats_file_int() { dx "$1" cat /tmp/corescope-ingestor-stats.json 2> /dev/null | json_int "$2"; } obs_at_least() { [ "$(stats_file_int "$1" obs_inserted)" -ge "$2" ] 2> /dev/null; } -no_subscriber_running() { ! dx "$1" pidof mosquitto_sub; } +no_subscriber_running() { dx "$1" pidof mosquitto_sub > /dev/null; [ $? -eq 1 ]; } ids_of() { dx "$1" awk '/^Uid:/{u=$2} /^Gid:/{g=$2} END{print u ":" g}' "/proc/$2/status"; } env_has() { dx "$1" sh -c "tr '\\0' '\\n' < /proc/$2/environ | grep -qx '$3'"; } broker_restarted() { local p; p=$(dx "$1" pidof mosquitto) && [ -n "$p" ] && [ "$p" != "$2" ]; } retained_present() { [ "$(dx "$1" mosquitto_sub -h localhost -t "$RETAINED_TOPIC" -C 1 -W 3 2> /dev/null)" = "$RETAINED_MSG" ]; } +is_uid_gid() { case "$1" in [0-9]*:[0-9]*) return 0 ;; *) return 1 ;; esac; } # Broker identity + persistence: publish a retained message, SIGTERM the broker # (it saves its database on exit), let supervisord restart it, then require the # retained message, the database owner and a clean log. -check_broker_persistence() { # container label - local c=$1 label=$2 want pid newpid +check_broker_persistence() { # container-id label + local c=$1 label=$2 want pid newpid got logs errors want=$(dx "$c" sh -c 'echo "$(id -u mosquitto):$(id -g mosquitto)"') + require_now "$label: image has a mosquitto user (uid:gid '$want')" is_uid_gid "$want" pid=$(dx "$c" pidof mosquitto) + require_now "$label: broker process found" test -n "$pid" check "$label: broker runs as the mosquitto user" "$want" "$(ids_of "$c" "$pid")" dx "$c" mosquitto_pub -h localhost -q 1 -r -t "$RETAINED_TOPIC" -m "$RETAINED_MSG" dx "$c" kill -TERM "$pid" - if ! wait_for "$WAIT_S" broker_restarted "$c" "$pid"; then - check "$label: supervisord restarted the broker" "yes" "no" - return - fi + require "$label: supervisord restarted the broker" "$WAIT_S" broker_restarted "$c" "$pid" newpid=$(dx "$c" pidof mosquitto) check "$label: restarted broker runs as the mosquitto user" "$want" "$(ids_of "$c" "$newpid")" - wait_for 60 retained_present "$c" - check "$label: retained message survives a broker restart" "$RETAINED_MSG" \ - "$(dx "$c" mosquitto_sub -h localhost -t "$RETAINED_TOPIC" -C 1 -W 5 2> /dev/null)" + if wait_until 60 retained_present "$c"; then + got=$RETAINED_MSG + else + got=$(dx "$c" mosquitto_sub -h localhost -t "$RETAINED_TOPIC" -C 1 -W 5 2> /dev/null) + fi + check "$label: retained message survives a broker restart" "$RETAINED_MSG" "$got" check "$label: persistence database owned by the broker user" "$want" \ "$(dx "$c" stat -c '%u:%g' /var/lib/mosquitto/mosquitto.db 2> /dev/null)" - check "$label: no persistence write errors logged" "0" \ - "$(docker logs "$c" 2>&1 | grep -cE 'Error saving|Permission denied')" + if logs=$(dlogs "$c" 2>&1); then + errors=$(printf '%s\n' "$logs" | grep -cE 'Error saving|Permission denied') + else + errors="unknown (docker logs failed)" + fi + check "$label: no persistence write errors logged" "0" "$errors" } -echo "# image: $IMAGE platform: ${PLATFORM:-default}" +# --------------------------------------------------------------------------- +# Setup: local image only, resolved once + +command -v docker > /dev/null 2>&1 || setup_error "docker CLI not found" +TMP=$(mktemp -d) || setup_error "mktemp failed" +trap on_exit EXIT +trap 'on_signal 1 HUP' HUP +trap 'on_signal 2 INT' INT +trap 'on_signal 15 TERM' TERM + +run_bounded "$EXEC_S" docker version -f '{{.Server.Version}}' > /dev/null 2>&1 || setup_error "docker daemon not reachable" +IMAGE_ID=$(run_bounded "$EXEC_S" docker image inspect -f '{{.Id}}' "$IMAGE" 2> /dev/null) +case "$IMAGE_ID" in + sha256:*) ;; + *) setup_error "image '$IMAGE' is not available locally (this test never pulls images)" ;; +esac +IMAGE_PLATFORM=$(run_bounded "$EXEC_S" docker image inspect -f '{{.Os}}/{{.Architecture}}' "$IMAGE_ID") +if [ -n "$PLATFORM" ] && [ "$(printf '%s' "$PLATFORM" | cut -d/ -f1-2)" != "$IMAGE_PLATFORM" ]; then + setup_error "image $IMAGE_ID is $IMAGE_PLATFORM, not $PLATFORM" +fi +echo "# run: $RUN_ID" +echo "# image: $IMAGE" +echo "# image id: $IMAGE_ID ($IMAGE_PLATFORM)" +echo "# limits: TOTAL_S=${TOTAL_S}s (incl. ${CLEANUP_RESERVE_S}s cleanup reserve), WAIT_S=${WAIT_S}s, EXEC_S=${EXEC_S}s" +echo "# logs: $LOG_DIR" # --------------------------------------------------------------------------- echo "# case 1: overlapping subscriptions, no PUID/PGID" -C1="corescope-$RUN_ID-overlap" mkdir -p "$TMP/overlap" cat > "$TMP/overlap/config.json" << 'EOF' {"dbPath": "/app/data/meshcore.db", "mqttSources": [{"name": "local", "broker": "mqtt://localhost:1883", "topics": ["meshcore/+/+/packets", "meshcore/#"], "connectTimeoutSec": 30}]} EOF -start_container "$C1" "$TMP/overlap/config.json" || { echo "FAIL - could not start $C1"; exit 1; } -wait_for "$WAIT_S" server_up "$C1" || check "case 1: server ready" "yes" "no" -wait_for "$WAIT_S" subscribed_at_least "$C1" 2 || check "case 1: ingestor subscribed to both topics" "2" "$(subscribed_topics "$C1")" +require_now "case 1: container created from $IMAGE_ID and started" start_container "corescope-$RUN_ID-overlap" "$TMP/overlap/config.json" +C1=$LAST_ID +require "case 1: server ready" "$WAIT_S" server_up "$C1" +require "case 1: ingestor subscribed to both topics" "$WAIT_S" subscribed_at_least "$C1" 2 +echo "# mosquitto: $(dx "$C1" sh -c 'mosquitto -h 2>&1 | head -1')" dx -d "$C1" sh -c "mosquitto_sub -h localhost -v -t 'meshcore/+/+/packets' -t 'meshcore/#' -W 20 > /tmp/compat-overlap.txt 2>&1" dx -d "$C1" sh -c "mosquitto_sub -h localhost -v -t 'meshcore/#' -W 20 > /tmp/compat-single.txt 2>&1" -sleep 5 +pause 5 # let both test subscribers connect before publishing dx "$C1" mosquitto_pub -h localhost -q 0 -t "$TOPIC" -m "$PAYLOAD" -wait_for "$WAIT_S" obs_at_least "$C1" 1 || check "case 1: ingestor stored the packet" "yes" "no" -wait_for 60 no_subscriber_running "$C1" -sleep 12 # > /api/stats cache TTL (10s); the ingestor stats file is rewritten every second +require "case 1: ingestor stored the packet" "$WAIT_S" obs_at_least "$C1" 1 +require "case 1: test subscribers finished" 60 no_subscriber_running "$C1" +pause 12 # > /api/stats cache TTL (10s); the ingestor stats file is rewritten every second check "case 1: deliveries to one client with overlapping subscriptions" "1" "$(dx "$C1" grep -c "^$TOPIC " /tmp/compat-overlap.txt)" check "case 1: deliveries to a client with a single subscription" "1" "$(dx "$C1" grep -c "^$TOPIC " /tmp/compat-single.txt)" -check "case 1: /api/mqtt/status packetsTotal" "1" "$(dx "$C1" wget -qO- http://localhost:3000/api/mqtt/status | json_int packetsTotal)" +check "case 1: /api/mqtt/status packetsTotal" "1" "$(dx "$C1" wget -T 5 -qO- http://localhost:3000/api/mqtt/status | json_int packetsTotal)" check "case 1: ingestor tx_inserted" "1" "$(stats_file_int "$C1" tx_inserted)" check "case 1: ingestor tx_dupes" "0" "$(stats_file_int "$C1" tx_dupes)" check "case 1: ingestor obs_inserted" "1" "$(stats_file_int "$C1" obs_inserted)" check "case 1: ingestor observer_upserts" "1" "$(stats_file_int "$C1" observer_upserts)" -STATS=$(dx "$C1" wget -qO- http://localhost:3000/api/stats) +STATS=$(dx "$C1" wget -T 5 -qO- http://localhost:3000/api/stats) check "case 1: /api/stats totalTransmissions" "1" "$(echo "$STATS" | json_int totalTransmissions)" check "case 1: /api/stats totalObservations" "1" "$(echo "$STATS" | json_int totalObservations)" check_broker_persistence "$C1" "case 1" -stop_container "$C1" +retire_container "$C1" # --------------------------------------------------------------------------- echo "# case 2: PUID=1000 PGID=1000 in /app/data/.env (exported by the entrypoint)" -C2="corescope-$RUN_ID-puid" mkdir -p "$TMP/puid" printf 'PUID=1000\nPGID=1000\n' > "$TMP/puid/.env" -start_container "$C2" "$TMP/puid/.env" || { echo "FAIL - could not start $C2"; exit 1; } -wait_for "$WAIT_S" server_up "$C2" || check "case 2: server ready" "yes" "no" -wait_for "$WAIT_S" subscribed_at_least "$C2" 1 || check "case 2: ingestor subscribed" "1" "$(subscribed_topics "$C2")" +require_now "case 2: container created from $IMAGE_ID and started" start_container "corescope-$RUN_ID-puid" "$TMP/puid/.env" +C2=$LAST_ID +require "case 2: server ready" "$WAIT_S" server_up "$C2" +require "case 2: ingestor subscribed" "$WAIT_S" subscribed_at_least "$C2" 1 check "case 2: supervisord (pid 1) keeps PUID/PGID" "yes" \ "$(env_has "$C2" 1 PUID=1000 && env_has "$C2" 1 PGID=1000 && echo yes || echo no)" check "case 2: supervisord still runs as root" "0:0" "$(ids_of "$C2" 1)" @@ -182,11 +475,10 @@ for p in corescope-ingestor corescope-server caddy; do check "case 2: $p still runs as root" "0:0" "$(ids_of "$C2" "$pid")" done dx "$C2" mosquitto_pub -h localhost -q 0 -t "$TOPIC" -m "$PAYLOAD" -wait_for "$WAIT_S" obs_at_least "$C2" 1 +require "case 2: ingestor stored the packet" "$WAIT_S" obs_at_least "$C2" 1 check "case 2: ingestor stored the packet via the broker" "1" "$(stats_file_int "$C2" tx_inserted)" check_broker_persistence "$C2" "case 2" -stop_container "$C2" +retire_container "$C2" -echo "# $CHECKS checks, $FAILS failed" -if [ "$FAILS" -eq 0 ]; then echo "RESULT: PASS"; else echo "RESULT: FAIL"; fi -[ "$FAILS" -eq 0 ] +COMPLETED=1 +finish From dcbdd90bf88ab0fdfe322aa07af2c26af985a538 Mon Sep 17 00:00:00 2001 From: Openclaw Date: Tue, 15 Sep 2026 11:02:00 +0200 Subject: [PATCH 3/4] test(docker): keep mosquitto compat test failures visible and fail-closed Follow-up to the independent review of 3c1b7ca1 (N1, N3, N5, N8). Only scripts/test-docker-mosquitto-compat.sh changes; the 28 checks, the 15 preconditions and their expected values are unchanged. - N1: stdout/stderr are saved on fds 3/4 at start and restored at the top of the signal and exit traps and of finish, so signal, RESULT and cleanup lines are printed even when a signal lands inside a redirected docker call. Child processes get fds 3/4 closed. - N3: if the case 1 container cannot be verifiably stopped and removed, the run stops with a non-zero status before case 2 is created. A later successful cleanup pass never clears that earlier failure. - N5: the exit path no longer waits unbounded on a background child. Own children get TERM, then KILL after a short grace period, and only while their PID still has the parent PID and start time recorded at spawn, so a recycled PID is never signalled. - N8: containers and volumes are present, absent or unknown. Absence is only concluded from a successful docker listing that no longer contains the object; failed or timed-out listings, inspects and post-removal checks are reported as CLEANUP UNRESOLVED and make the run non-zero, without any broader fallback deletion. Cleanup now runs before the RESULT line, and a verified cleanup is part of exit 0. SIGKILL still cannot run any of this, so the header keeps recommending an outer timeout. Co-Authored-By: Claude Opus 5 --- scripts/test-docker-mosquitto-compat.sh | 276 ++++++++++++++++++------ 1 file changed, 208 insertions(+), 68 deletions(-) diff --git a/scripts/test-docker-mosquitto-compat.sh b/scripts/test-docker-mosquitto-compat.sh index b80c502ca..fad852d1a 100755 --- a/scripts/test-docker-mosquitto-compat.sh +++ b/scripts/test-docker-mosquitto-compat.sh @@ -30,10 +30,18 @@ # and com.corescope.test.run=. On normal exit, failures and # SIGINT/SIGTERM/SIGHUP the test keeps each container's logs, removes exactly # its own labelled containers and their anonymous volumes, and verifies they -# are gone. SIGKILL cannot run that cleanup, so still run the test under an -# outer timeout; leftovers of a killed run can be listed with +# are gone. A container or volume only counts as gone when a successful docker +# listing no longer contains it; when docker cannot answer, cleanup is reported +# as unresolved and the run fails. If the case 1 container cannot be removed, +# case 2 is never started. Status, RESULT and cleanup lines go to the original +# stdout even when a signal interrupts a redirected docker call, and child +# processes that ignore TERM are killed after a short grace period. SIGKILL +# cannot run any of this, so still run the test under an outer timeout; +# leftovers of a killed run can be listed with # docker ps -a --filter label=com.corescope.test.run= # +# Requires bash, the docker CLI and ps supporting -o ppid,lstart,stat. +# # Environment (wall-clock seconds): # TOTAL_S budget for the whole run, including a 45s cleanup reserve (default 900) # WAIT_S limit for each readiness/progress wait (default 240) @@ -41,11 +49,15 @@ # LOG_DIR where container logs are kept # (default ${TMPDIR:-/tmp}/corescope-mqtt-compat-) # -# Exit status: 0 all 28 checks and all 15 preconditions ran and passed; -# 1 a check or precondition failed, the budget ran out or the run was incomplete; -# 2 setup error (bad arguments, no docker, image not local, platform mismatch); -# 3 all checks passed but cleanup failed; 128+N stopped by signal N. +# Exit status: 0 all 28 checks and all 15 preconditions ran and passed and +# cleanup was verified; 1 a check or precondition failed, the budget ran out, or +# the run stopped early or was incomplete; 2 setup error (bad arguments, no +# docker, image not local, platform mismatch, unsuitable ps); 3 all checks +# passed but cleanup failed or could not be verified; 128+N stopped by signal N. set -uo pipefail +# Keep the original stdout/stderr: the traps write through them even when a +# signal lands inside a function call whose output is redirected. +exec 3>&1 4>&2 IMAGE=${1:-} PLATFORM=${2:-} @@ -71,13 +83,15 @@ PRECONDITIONS=0 PRECONDITION_FAILS=0 COMPLETED=0 BUDGET_EXHAUSTED=0 -CLEANUP_FAILED=0 +CLEANUP_FAILED=0 # an earlier cleanup step failed; a later successful pass never clears it +CLEANUP_DONE=0 RESULT_PRINTED=0 IN_CLEANUP=0 OWNED="" # IDs of the containers this run created OWNED_VOLUMES="" # anonymous volumes those containers were created with LAST_ID="" -BG_PID="" +BG_PID="" # background child of this shell, with the ps identity recorded at spawn +BG_IDENT="" TMP="" setup_error() { @@ -95,25 +109,64 @@ CLEANUP_DEADLINE=$((SECONDS + TOTAL_S)) # --------------------------------------------------------------------------- # Wall-clock bounded execution +# A process identity is its parent PID plus start time as reported by ps. +# Signals are only sent while a PID still has the identity recorded when this +# script spawned it, so a recycled PID is never signalled. +proc_identity() { ps -o ppid= -o lstart= -p "$1" 2> /dev/null | tr -s ' '; } +proc_zombie() { case "$(ps -o stat= -p "$1" 2> /dev/null)" in *Z*) return 0 ;; *) return 1 ;; esac; } +proc_alive() { [ -n "$2" ] && [ "$(proc_identity "$1")" = "$2" ] && ! proc_zombie "$1"; } + +# stop_own_child : TERM, KILL after seconds, then +# reap. Never waits unbounded: returns 1 if the process is still there 2s after +# KILL, or if it cannot be verified as ours (no recorded identity). +stop_own_child() { + local pid=$1 ident=$2 grace=$3 end + [ -n "$pid" ] || return 0 + if ! proc_alive "$pid" "$ident"; then + if kill -0 "$pid" 2> /dev/null && ! proc_zombie "$pid"; then + [ -n "$ident" ] && return 0 # the PID now belongs to another process: not ours + return 1 # no recorded identity: do not signal, report it + fi + wait "$pid" 2> /dev/null + return 0 + fi + kill -TERM "$pid" 2> /dev/null + end=$((SECONDS + grace)) + while proc_alive "$pid" "$ident"; do + if [ "$SECONDS" -ge "$end" ]; then + kill -KILL "$pid" 2> /dev/null + break + fi + sleep 0.2 + done + end=$((SECONDS + 2)) + while proc_alive "$pid" "$ident"; do + [ "$SECONDS" -lt "$end" ] || return 1 + sleep 0.2 + done + wait "$pid" 2> /dev/null + return 0 +} + # run_bounded : run an external command with a wall-clock # limit, capped by what is left of the run's budget. Returns 124 on timeout. run_bounded() { - local limit=$1 left end pid rc + local limit=$1 left end pid ident rc shift if [ "$IN_CLEANUP" = 1 ]; then left=$((CLEANUP_DEADLINE - SECONDS)); else left=$((DEADLINE - SECONDS)); fi [ "$left" -ge "$limit" ] || limit=$left [ "$limit" -gt 0 ] || return 124 - "$@" & + "$@" 3>&- 4>&- & pid=$! + ident=$(proc_identity "$pid") BG_PID=$pid + BG_IDENT=$ident end=$((SECONDS + limit)) while kill -0 "$pid" 2> /dev/null; do if [ "$SECONDS" -ge "$end" ]; then - kill -TERM "$pid" 2> /dev/null - sleep 1 - kill -KILL "$pid" 2> /dev/null - wait "$pid" 2> /dev/null + stop_own_child "$pid" "$ident" 2 BG_PID="" + BG_IDENT="" return 124 fi sleep 0.2 @@ -121,6 +174,7 @@ run_bounded() { wait "$pid" rc=$? BG_PID="" + BG_IDENT="" return "$rc" } @@ -128,10 +182,12 @@ pause() { # : interruptible sleep, capped by the run's budget local s=$1 left=$((DEADLINE - SECONDS)) [ "$left" -ge "$s" ] || s=$left [ "$s" -gt 0 ] || return 0 - sleep "$s" & + sleep "$s" 3>&- 4>&- & BG_PID=$! + BG_IDENT=$(proc_identity "$BG_PID") wait "$BG_PID" BG_PID="" + BG_IDENT="" } wait_until() { # : poll until the predicate succeeds; 1 on timeout @@ -147,20 +203,45 @@ wait_until() { # : poll until the predicate succeeds; 1 on # --------------------------------------------------------------------------- # Results +begin_cleanup() { # once: ignore further signals and switch to the cleanup reserve + [ "$IN_CLEANUP" = 1 ] && return 0 + trap '' INT TERM HUP + IN_CLEANUP=1 + CLEANUP_DEADLINE=$((SECONDS + CLEANUP_RESERVE_S)) +} + +cleanup_summary() { # + local note="" + [ "$CLEANUP_FAILED" = 0 ] || note=" (an earlier cleanup step failed)" + echo "# checks: $CHECKS of $EXPECTED_CHECKS run, $FAILS failed; preconditions: $((PRECONDITIONS - PRECONDITION_FAILS)) of $EXPECTED_PRECONDITIONS met; cleanup: $1$note" +} + finish() { - local met=$((PRECONDITIONS - PRECONDITION_FAILS)) - echo "# checks: $CHECKS of $EXPECTED_CHECKS run, $FAILS failed; preconditions: $met of $EXPECTED_PRECONDITIONS met" + local met=$((PRECONDITIONS - PRECONDITION_FAILS)) state=verified + exec 1>&3 2>&4 + begin_cleanup + cleanup || state="FAILED or unresolved" + CLEANUP_DONE=1 + cleanup_summary "$state" RESULT_PRINTED=1 + if [ "$COMPLETED" != 1 ]; then + if [ "$FAILS" -ne 0 ] || [ "$PRECONDITION_FAILS" -ne 0 ] || [ "$BUDGET_EXHAUSTED" -ne 0 ] || [ "$CLEANUP_FAILED" -ne 0 ]; then + echo "RESULT: FAIL (run incomplete)" + else + echo "RESULT: INCOMPLETE" + fi + exit 1 + fi if [ "$FAILS" -ne 0 ] || [ "$PRECONDITION_FAILS" -ne 0 ] || [ "$BUDGET_EXHAUSTED" -ne 0 ]; then - if [ "$COMPLETED" = 1 ]; then echo "RESULT: FAIL"; else echo "RESULT: FAIL (run incomplete)"; fi + echo "RESULT: FAIL" exit 1 fi - if [ "$COMPLETED" != 1 ] || [ "$CHECKS" -ne "$EXPECTED_CHECKS" ] || [ "$met" -ne "$EXPECTED_PRECONDITIONS" ]; then + if [ "$CHECKS" -ne "$EXPECTED_CHECKS" ] || [ "$met" -ne "$EXPECTED_PRECONDITIONS" ]; then echo "RESULT: INCOMPLETE" exit 1 fi - if [ "$CLEANUP_FAILED" -ne 0 ]; then - echo "RESULT: FAIL (all checks passed, cleanup failed)" + if [ "$state" != verified ] || [ "$CLEANUP_FAILED" -ne 0 ]; then + echo "RESULT: FAIL (all checks passed, cleanup failed or unresolved)" exit 3 fi echo "RESULT: PASS" @@ -220,8 +301,23 @@ require_now() { # description command...: one attempt, stop the run if it fails volumes_of() { run_bounded "$EXEC_S" docker container inspect -f '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}} {{end}}{{end}}' "$1"; } +# container_state / volume_state : prints present, absent or unknown. +# Absence is only concluded from a successful listing that does not contain the +# object; a failed or timed-out docker command yields unknown. +container_state() { + local out + out=$(run_bounded "$EXEC_S" docker ps -a -q --no-trunc) || { echo unknown; return; } + if printf '%s\n' "$out" | grep -qxF "$1"; then echo present; else echo absent; fi +} + +volume_state() { + local out + out=$(run_bounded "$EXEC_S" docker volume ls -q) || { echo unknown; return; } + if printf '%s\n' "$out" | grep -qxF "$1"; then echo present; else echo absent; fi +} + start_container() { # name [file copied into /app/data ...]; sets LAST_ID - local name=$1 id f + local name=$1 id vols f shift local plat=() [ -n "$PLATFORM" ] && plat=(--platform "$PLATFORM") @@ -235,7 +331,11 @@ start_container() { # name [file copied into /app/data ...]; sets LAST_ID fi OWNED="$OWNED $id" LAST_ID=$id - OWNED_VOLUMES="$OWNED_VOLUMES $(volumes_of "$id")" + if ! vols=$(volumes_of "$id"); then + echo "# could not read the volumes of $name" + return 1 + fi + OWNED_VOLUMES="$OWNED_VOLUMES $vols" if [ "$(run_bounded "$EXEC_S" docker container inspect -f '{{.Image}}' "$id")" != "$IMAGE_ID" ]; then echo "# $name was not created from $IMAGE_ID" return 1 @@ -252,99 +352,139 @@ start_container() { # name [file copied into /app/data ...]; sets LAST_ID fi } -remove_volume() { # anonymous volume created with one of our containers - local v=$1 rc users labels - run_bounded "$EXEC_S" docker volume inspect "$v" > /dev/null 2>&1 - rc=$? - [ "$rc" -ne 124 ] || { echo "CLEANUP FAIL - timed out inspecting volume $v"; return 1; } - [ "$rc" -eq 0 ] || return 0 - labels=$(run_bounded "$EXEC_S" docker volume inspect -f '{{json .Labels}}' "$v") +remove_volume() { # anonymous volume recorded from one of this run's containers + local v=$1 labels users + case "$(volume_state "$v")" in + absent) return 0 ;; + present) ;; + *) echo "CLEANUP UNRESOLVED - could not determine whether volume $v still exists; left untouched"; return 1 ;; + esac + if ! labels=$(run_bounded "$EXEC_S" docker volume inspect -f '{{json .Labels}}' "$v"); then + echo "CLEANUP UNRESOLVED - could not inspect volume $v; left untouched" + return 1 + fi case "$labels" in *'"com.docker.volume.anonymous"'*) ;; *) echo "CLEANUP SKIP - volume $v is not marked anonymous; left untouched"; return 1 ;; esac - users=$(run_bounded "$EXEC_S" docker ps -a -q --filter "volume=$v") + if ! users=$(run_bounded "$EXEC_S" docker ps -a -q --no-trunc --filter "volume=$v"); then + echo "CLEANUP UNRESOLVED - could not list the users of volume $v; left untouched" + return 1 + fi if [ -n "$users" ]; then echo "CLEANUP SKIP - volume $v is still used by $users; left untouched" return 1 fi run_bounded "$EXEC_S" docker volume rm "$v" > /dev/null 2>&1 - run_bounded "$EXEC_S" docker volume inspect "$v" > /dev/null 2>&1 - [ $? -eq 1 ] || { echo "CLEANUP FAIL - volume $v still exists or could not be checked"; return 1; } + case "$(volume_state "$v")" in + absent) return 0 ;; + present) echo "CLEANUP FAIL - volume $v still exists"; return 1 ;; + *) echo "CLEANUP UNRESOLVED - could not confirm that volume $v is gone"; return 1 ;; + esac } remove_container() { # container id: verify the run label, keep logs, remove it and its anonymous volumes - local id=$1 rc label name vols v - run_bounded "$EXEC_S" docker container inspect "$id" > /dev/null 2>&1 - rc=$? - [ "$rc" -ne 124 ] || { echo "CLEANUP FAIL - timed out inspecting container $id"; return 1; } - [ "$rc" -eq 0 ] || return 0 - label=$(run_bounded "$EXEC_S" docker container inspect -f "{{index .Config.Labels \"$LABEL_RUN\"}}" "$id") + local id=$1 label name vols vols_known=1 v ok=0 + case "$(container_state "$id")" in + absent) echo "# container $id is already gone"; return 0 ;; + present) ;; + *) echo "CLEANUP UNRESOLVED - could not determine whether container $id still exists; left untouched"; return 1 ;; + esac + if ! label=$(run_bounded "$EXEC_S" docker container inspect -f "{{index .Config.Labels \"$LABEL_RUN\"}}" "$id"); then + echo "CLEANUP UNRESOLVED - could not read the labels of container $id; left untouched" + return 1 + fi if [ "$label" != "$RUN_ID" ]; then echo "CLEANUP SKIP - container $id is not labelled $LABEL_RUN=$RUN_ID; left untouched" return 1 fi name=$(run_bounded "$EXEC_S" docker container inspect -f '{{.Name}}' "$id" | tr -d '/') - vols=$(volumes_of "$id") + vols=$(volumes_of "$id") || vols_known=0 if mkdir -p "$LOG_DIR" 2> /dev/null; then - run_bounded "$EXEC_S" docker logs -t "$id" > "$LOG_DIR/${name:-$id}.log" 2>&1 + run_bounded "$EXEC_S" docker logs -t "$id" > "$LOG_DIR/${name:-$id}.log" 2>&1 || + echo "# could not save the logs of ${name:-$id}" fi run_bounded 20 docker stop -t 10 "$id" > /dev/null 2>&1 run_bounded "$EXEC_S" docker rm -f -v "$id" > /dev/null 2>&1 - run_bounded "$EXEC_S" docker container inspect "$id" > /dev/null 2>&1 - [ $? -eq 1 ] || { echo "CLEANUP FAIL - container ${name:-$id} still exists or could not be checked"; return 1; } + case "$(container_state "$id")" in + absent) ;; + present) echo "CLEANUP FAIL - container ${name:-$id} still exists"; return 1 ;; + *) echo "CLEANUP UNRESOLVED - could not confirm that container ${name:-$id} is gone"; return 1 ;; + esac + if [ "$vols_known" = 0 ]; then + echo "CLEANUP UNRESOLVED - removed container ${name:-$id}, but could not list its volumes to verify them" + return 1 + fi for v in $vols; do - remove_volume "$v" || return 1 + remove_volume "$v" || ok=1 done + [ "$ok" = 0 ] || return 1 echo "# removed container ${name:-$id} and its anonymous volumes:" $vols } -retire_container() { # container id: remove it now and forget it +forget_container() { # container id local x rest="" + for x in $OWNED; do [ "$x" = "$1" ] || rest="$rest $x"; done + OWNED=$rest +} + +retire_container() { # container id, label: it must be verifiably gone before anything else starts if remove_container "$1"; then - for x in $OWNED; do [ "$x" = "$1" ] || rest="$rest $x"; done - OWNED=$rest + forget_container "$1" else CLEANUP_FAILED=1 + echo "FAIL - [sequence] $2 container could not be stopped and removed safely; no further containers are started" + finish fi } -cleanup() { +cleanup() { # one cleanup pass; status 1 if anything failed or stayed unresolved in this pass local ok=0 id v listed - listed=$(run_bounded "$EXEC_S" docker ps -a -q --no-trunc --filter "label=$LABEL_RUN=$RUN_ID") + if ! listed=$(run_bounded "$EXEC_S" docker ps -a -q --no-trunc --filter "label=$LABEL_RUN=$RUN_ID"); then + echo "CLEANUP UNRESOLVED - could not list containers labelled $LABEL_RUN=$RUN_ID" + listed="" + ok=1 + fi for id in $(printf '%s\n' $OWNED $listed | sort -u); do - remove_container "$id" || ok=1 + if remove_container "$id"; then forget_container "$id"; else ok=1; fi done for v in $(printf '%s\n' $OWNED_VOLUMES | sort -u); do remove_volume "$v" || ok=1 done - [ "$CLEANUP_FAILED" = 0 ] || ok=1 return "$ok" } on_signal() { # number name + exec 1>&3 2>&4 echo "# received SIG$2; stopping and cleaning up" exit $((128 + $1)) } on_exit() { - local rc=$? - trap '' INT TERM HUP - IN_CLEANUP=1 - CLEANUP_DEADLINE=$((SECONDS + CLEANUP_RESERVE_S)) + local rc=$? state=verified + exec 1>&3 2>&4 + begin_cleanup if [ -n "$BG_PID" ]; then - kill -TERM "$BG_PID" 2> /dev/null - wait "$BG_PID" 2> /dev/null - fi - if [ "$RESULT_PRINTED" != 1 ]; then - echo "# checks: $CHECKS of $EXPECTED_CHECKS run, $FAILS failed; preconditions: $((PRECONDITIONS - PRECONDITION_FAILS)) of $EXPECTED_PRECONDITIONS met" - echo "RESULT: INCOMPLETE" - [ "$rc" -ne 0 ] || rc=1 + if ! stop_own_child "$BG_PID" "$BG_IDENT" 3; then + echo "CLEANUP UNRESOLVED - own child process $BG_PID could not be stopped and verified" + CLEANUP_FAILED=1 + fi + BG_PID="" + BG_IDENT="" fi - if ! cleanup; then - echo "CLEANUP FAIL - check for leftovers labelled $LABEL_RUN=$RUN_ID" - [ "$rc" -ne 0 ] || rc=3 + if [ "$CLEANUP_DONE" != 1 ]; then + cleanup || state="FAILED or unresolved" + CLEANUP_DONE=1 + if [ "$RESULT_PRINTED" != 1 ]; then + cleanup_summary "$state" + echo "RESULT: INCOMPLETE" + [ "$rc" -ne 0 ] || rc=1 + elif [ "$state" != verified ]; then + echo "CLEANUP FAIL - leftovers may carry the label $LABEL_RUN=$RUN_ID" + [ "$rc" -ne 0 ] || rc=3 + fi fi + if [ "$CLEANUP_FAILED" != 0 ] && [ "$rc" -eq 0 ]; then rc=3; fi [ -z "$TMP" ] || rm -rf "$TMP" exit "$rc" } @@ -402,6 +542,7 @@ check_broker_persistence() { # container-id label # Setup: local image only, resolved once command -v docker > /dev/null 2>&1 || setup_error "docker CLI not found" +ps -o ppid= -o lstart= -o stat= -p $$ > /dev/null 2>&1 || setup_error "ps with -o ppid,lstart,stat is required to bound child processes" TMP=$(mktemp -d) || setup_error "mktemp failed" trap on_exit EXIT trap 'on_signal 1 HUP' HUP @@ -455,7 +596,7 @@ STATS=$(dx "$C1" wget -T 5 -qO- http://localhost:3000/api/stats) check "case 1: /api/stats totalTransmissions" "1" "$(echo "$STATS" | json_int totalTransmissions)" check "case 1: /api/stats totalObservations" "1" "$(echo "$STATS" | json_int totalObservations)" check_broker_persistence "$C1" "case 1" -retire_container "$C1" +retire_container "$C1" "case 1" # --------------------------------------------------------------------------- echo "# case 2: PUID=1000 PGID=1000 in /app/data/.env (exported by the entrypoint)" @@ -478,7 +619,6 @@ dx "$C2" mosquitto_pub -h localhost -q 0 -t "$TOPIC" -m "$PAYLOAD" require "case 2: ingestor stored the packet" "$WAIT_S" obs_at_least "$C2" 1 check "case 2: ingestor stored the packet via the broker" "1" "$(stats_file_int "$C2" tx_inserted)" check_broker_persistence "$C2" "case 2" -retire_container "$C2" - COMPLETED=1 +retire_container "$C2" "case 2" finish From 2f3f03dda25110f8cf11d3ca465145836df471c9 Mon Sep 17 00:00:00 2001 From: Openclaw Date: Wed, 16 Sep 2026 09:36:46 +0200 Subject: [PATCH 4/4] test(docker): read the whole listing when deciding container/volume state Follow-up to the independent review of dcbdd90b (blocker B1). Only scripts/test-docker-mosquitto-compat.sh changes; the 28 checks, the 15 preconditions and their expected values are unchanged. container_state and volume_state matched with printf ... | grep -qxF. grep -q exits at the first hit, the writing printf then gets SIGPIPE, and pipefail turns that hit into a non-zero pipeline, so on a host whose docker ps -a -q or docker volume ls -q listing is large enough an existing container or volume was reported as absent: cleanup skipped it, "already gone" was printed for a container that was still running, and case 2 could be started on top of case 1. The match now reads the whole listing, and the two status sources are kept apart: writer 0 and grep 0 is present, writer 0 and grep 1 is absent, and anything else - a failed writer or a grep error - is unknown, which stays CLEANUP UNRESOLVED and makes the run non-zero. A failed or timed-out listing is still unknown as before. Co-Authored-By: Claude Opus 5 --- scripts/test-docker-mosquitto-compat.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/test-docker-mosquitto-compat.sh b/scripts/test-docker-mosquitto-compat.sh index fad852d1a..2c6e876ce 100755 --- a/scripts/test-docker-mosquitto-compat.sh +++ b/scripts/test-docker-mosquitto-compat.sh @@ -303,17 +303,26 @@ volumes_of() { run_bounded "$EXEC_S" docker container inspect -f '{{range .Mount # container_state / volume_state : prints present, absent or unknown. # Absence is only concluded from a successful listing that does not contain the -# object; a failed or timed-out docker command yields unknown. +# object; a failed or timed-out docker command yields unknown. The match has to +# read the whole listing: grep -q exits at the first hit, the writing printf +# then gets SIGPIPE and pipefail turns that hit into a non-zero pipeline, so on +# a host with many containers or volumes a present object looked absent. Only a +# clean no-match (writer 0, grep 1) is absence; a failed writer or a grep error +# is unknown, never absent. container_state() { - local out + local out st out=$(run_bounded "$EXEC_S" docker ps -a -q --no-trunc) || { echo unknown; return; } - if printf '%s\n' "$out" | grep -qxF "$1"; then echo present; else echo absent; fi + printf '%s\n' "$out" | grep -xF -- "$1" > /dev/null + st="${PIPESTATUS[0]}:${PIPESTATUS[1]}" + case "$st" in 0:0) echo present ;; 0:1) echo absent ;; *) echo unknown ;; esac } volume_state() { - local out + local out st out=$(run_bounded "$EXEC_S" docker volume ls -q) || { echo unknown; return; } - if printf '%s\n' "$out" | grep -qxF "$1"; then echo present; else echo absent; fi + printf '%s\n' "$out" | grep -xF -- "$1" > /dev/null + st="${PIPESTATUS[0]}:${PIPESTATUS[1]}" + case "$st" in 0:0) echo present ;; 0:1) echo absent ;; *) echo unknown ;; esac } start_container() { # name [file copied into /app/data ...]; sets LAST_ID