From 1494a4a5ac3a99498cf196a6718ce97b5c26eb1e Mon Sep 17 00:00:00 2001 From: Han Ngo Date: Wed, 18 Nov 2026 10:16:06 +0700 Subject: [PATCH 1/3] feat(memory-soak): ping a vps-mon heartbeat after each sample --- scripts/memory-soak/sample | 16 ++++++++++- scripts/memory-soak/test-heartbeat | 45 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100755 scripts/memory-soak/test-heartbeat diff --git a/scripts/memory-soak/sample b/scripts/memory-soak/sample index 2f8090c5..fba4a91e 100755 --- a/scripts/memory-soak/sample +++ b/scripts/memory-soak/sample @@ -5,7 +5,12 @@ # # sample [END_EPOCH] END_EPOCH: unix seconds. Past it, boot the LaunchAgent out and exit. # -# Env: HB_SOAK_LOG (default ~/Library/Logs/hiddenbar-memory-soak.log), HB_SOAK_APP, HB_SOAK_LABEL. +# Env: HB_SOAK_LOG (default ~/Library/Logs/hiddenbar-memory-soak.log), HB_SOAK_APP, HB_SOAK_LABEL, +# HB_SOAK_HB_FILE and HB_SOAK_PING (the vps-mon heartbeat URL file and the hb-ping helper). +# +# Liveness: each completed sample (app running or not) pings the mini.hiddenbar-memory-soak heartbeat. +# No URL file or a failed ping never changes the exit code. The end-epoch branch does NOT ping, so the +# heartbeat goes silent after the soak by design: retire it with vps-mon bin/retire-heartbeat then. # Line: TS pid=N up=ETIME footprint=B peak=B nodes=N heap_bytes=B version=SHORT+BUILD set -u @@ -13,10 +18,17 @@ LOG="${HB_SOAK_LOG:-$HOME/Library/Logs/hiddenbar-memory-soak.log}" APP="${HB_SOAK_APP:-/Applications/Hidden Bar.app}" LABEL="${HB_SOAK_LABEL:-mini.hiddenbar-memory-soak}" END_EPOCH="${1:-}" +HB_FILE="${HB_SOAK_HB_FILE:-$HOME/.config/vps-mon/hiddenbar-memory-soak-heartbeat-url}" +HB_PING="${HB_SOAK_PING:-$HOME/workspace/dwarvesf/vps-mon/bin/hb-ping}" mkdir -p "$(dirname "$LOG")" ts() { date +%Y-%m-%dT%H:%M:%S%z; } log() { printf '%s %s\n' "$(ts)" "$*" >> "$LOG"; } +heartbeat() { + if [ -r "$HB_FILE" ] && [ -x "$HB_PING" ]; then + "$HB_PING" "$HB_FILE" || log "heartbeat ping failed" + fi +} if [ -n "$END_EPOCH" ] && [ "$(date +%s)" -ge "$END_EPOCH" ]; then log "state=soak-ended (end date reached, booting out $LABEL)" @@ -36,6 +48,7 @@ version="$short+$build" pid="$(pgrep -f "$APP/Contents/MacOS/Hidden Bar" | head -n 1)" if [ -z "$pid" ]; then log "pid=- state=not-running version=$version" + heartbeat exit 0 fi @@ -59,3 +72,4 @@ if [ -z "$nodes" ]; then fi log "pid=$pid up=$up footprint=$footprint peak=$peak nodes=$nodes heap_bytes=$heap_bytes version=$version$note" +heartbeat diff --git a/scripts/memory-soak/test-heartbeat b/scripts/memory-soak/test-heartbeat new file mode 100755 index 00000000..bcad619c --- /dev/null +++ b/scripts/memory-soak/test-heartbeat @@ -0,0 +1,45 @@ +#!/bin/bash +# scripts/memory-soak/test-heartbeat : prove `sample` pings its heartbeat once per completed sample and not otherwise. +# +# Uses a stub ping helper that appends a line per call, a temp log, and a label that is not loaded, so the +# live LaunchAgent, the live log and the live heartbeat are never touched. Cases: +# sampled app not running (no such app path), URL file present -> 1 ping, exit 0 +# no-url URL file absent -> 0 pings, exit 0 +# soak-ended END_EPOCH already past (NEGATIVE CONTROL) -> 0 pings, exit 0 +# ping-fails ping helper exits non-zero -> exit 0, "heartbeat ping failed" logged +# Exit 0 when every case matches, 1 otherwise. +set -u + +HERE="$(cd "$(dirname "$0")" && pwd)" +SAMPLE="$HERE/sample" +T="$(mktemp -d)" +fails=0 + +printf '#!/bin/bash\necho ping >> "%s/hits"\n' "$T" > "$T/ping-ok" +printf '#!/bin/bash\nexit 7\n' > "$T/ping-bad" +chmod +x "$T/ping-ok" "$T/ping-bad" +echo "https://example.invalid/v1/heartbeat/FIXTURE" > "$T/url" + +# run +run() { + : >| "$T/hits" + : >| "$T/log" + HB_SOAK_LOG="$T/log" HB_SOAK_APP="$T/NoSuch.app" HB_SOAK_LABEL=mini.hiddenbar-memory-soak-test \ + HB_SOAK_HB_FILE="$3" HB_SOAK_PING="$2" "$SAMPLE" $4 >/dev/null 2>&1 + local rc=$? pings + pings=$(wc -l < "$T/hits" | tr -d ' ') + if [ "$rc" = 0 ] && [ "$pings" = "$5" ] && { [ -z "$6" ] || grep -q "$6" "$T/log"; }; then + printf 'PASS %-11s pings=%s exit=%s\n' "$1" "$pings" "$rc" + else + printf 'FAIL %-11s pings=%s exit=%s (want pings=%s exit=0 log~%s)\n' "$1" "$pings" "$rc" "$5" "$6" + fails=$((fails + 1)) + fi +} + +run sampled "$T/ping-ok" "$T/url" "" 1 "state=not-running" +run no-url "$T/ping-ok" "$T/none" "" 0 "state=not-running" +run soak-ended "$T/ping-ok" "$T/url" 1 0 "soak-ended" +run ping-fails "$T/ping-bad" "$T/url" "" 0 "heartbeat ping failed" + +[ "$fails" = 0 ] && echo "-- summary: all cases match" || echo "-- summary: $fails case(s) FAIL" +[ "$fails" = 0 ] From 9cc3caee08d9c38d58cc18831655dc4a0637e401 Mon Sep 17 00:00:00 2001 From: Han Ngo Date: Wed, 18 Nov 2026 10:16:28 +0700 Subject: [PATCH 2/3] docs(memory-soak): heartbeat proof with negative control --- docs/verification/memory-soak-heartbeat.md | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 docs/verification/memory-soak-heartbeat.md diff --git a/docs/verification/memory-soak-heartbeat.md b/docs/verification/memory-soak-heartbeat.md new file mode 100644 index 00000000..8bfb8a02 --- /dev/null +++ b/docs/verification/memory-soak-heartbeat.md @@ -0,0 +1,39 @@ +# memory-soak heartbeat: proof + +| | | +|---|---| +| Change | `scripts/memory-soak/sample` pings the vps-mon heartbeat `mini.hiddenbar-memory-soak` after each completed sample | +| Why | the LaunchAgent was a catalog gap (scheduled job, no monitor); a heartbeat is its liveness signal | +| Live after | merge, pull of the Mini `hidden` checkout, heartbeat provisioned (see PR) | + +## Green run + +- Command: `scripts/memory-soak/test-heartbeat` +- Exit: 0 +- Verdict: PASS + +``` +PASS sampled pings=1 exit=0 +PASS no-url pings=0 exit=0 +PASS soak-ended pings=0 exit=0 +PASS ping-fails pings=0 exit=0 +-- summary: all cases match +``` + +Real sample, stub ping helper, temp log, the installed Hidden Bar running: exit 0, one ping, one log line (`pid=... footprint=... nodes=...`). + +## NEGATIVE CONTROL + +- Command: `negctl.sh` with `sample` swapped for the pre-change copy (`git show HEAD~1:scripts/memory-soak/sample`) +- Exit: 1 under mutation, RED as expected: `FAIL sampled pings=0 (want 1)` and `FAIL ping-fails` (no "heartbeat ping failed" logged) +- Restored: exit 0, Verdict: PASS + +## Reproduce + +```bash scripts/memory-soak/test-heartbeat +scripts/memory-soak/test-heartbeat +``` + +## End of soak + +The `soak-ended` case is the proof the end-date branch does not ping: the heartbeat then goes silent on purpose. Retire it after the end epoch with vps-mon `bin/retire-heartbeat mini.hiddenbar-memory-soak`. From 7a6d72ffea6357de8094d3814e542d1478a9205f Mon Sep 17 00:00:00 2001 From: Han Ngo Date: Wed, 18 Nov 2026 10:28:27 +0700 Subject: [PATCH 3/3] fix(memory-soak): skip the ping without a footprint, cover a running app A sample with footprint=NA measured nothing, so it no longer pings. test-heartbeat gains a fake running app (real footprint) and a stubbed-footprint case; README gets the heartbeat and retirement note. --- docs/verification/memory-soak-heartbeat.md | 6 ++++++ scripts/memory-soak/README.md | 4 ++++ scripts/memory-soak/sample | 5 +++-- scripts/memory-soak/test-heartbeat | 20 ++++++++++++++++++-- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/verification/memory-soak-heartbeat.md b/docs/verification/memory-soak-heartbeat.md index 8bfb8a02..a3ec8c88 100644 --- a/docs/verification/memory-soak-heartbeat.md +++ b/docs/verification/memory-soak-heartbeat.md @@ -17,11 +17,15 @@ PASS sampled pings=1 exit=0 PASS no-url pings=0 exit=0 PASS soak-ended pings=0 exit=0 PASS ping-fails pings=0 exit=0 +PASS running pings=1 exit=0 +PASS no-footprint pings=0 exit=0 -- summary: all cases match ``` Real sample, stub ping helper, temp log, the installed Hidden Bar running: exit 0, one ping, one log line (`pid=... footprint=... nodes=...`). +`running` uses a fake "Hidden Bar" process and the real `footprint`; `no-footprint` stubs `footprint` to print nothing, so the sample logs `footprint=NA` and must not ping. + ## NEGATIVE CONTROL - Command: `negctl.sh` with `sample` swapped for the pre-change copy (`git show HEAD~1:scripts/memory-soak/sample`) @@ -37,3 +41,5 @@ scripts/memory-soak/test-heartbeat ## End of soak The `soak-ended` case is the proof the end-date branch does not ping: the heartbeat then goes silent on purpose. Retire it after the end epoch with vps-mon `bin/retire-heartbeat mini.hiddenbar-memory-soak`. + +Second control, review fixes: `negctl.sh` with the closing ping line deleted (`sed -i '' '/ || heartbeat$/d' scripts/memory-soak/sample`): `running` goes RED (pings=0, want 1). With `sample` at the previous commit (no footprint=NA skip): `no-footprint` goes RED (pings=1, want 0). diff --git a/scripts/memory-soak/README.md b/scripts/memory-soak/README.md index 2c28c00d..bd03cc1e 100644 --- a/scripts/memory-soak/README.md +++ b/scripts/memory-soak/README.md @@ -27,6 +27,10 @@ scripts/memory-soak/install --uninstall # stop early; the plist goes to the Tra The LaunchAgent is `mini.hiddenbar-memory-soak`. It runs at load and every 3600 s, and boots itself out once the end date (install time + 14 days, passed as an epoch argument) has passed. Check with `launchctl print gui/$UID/mini.hiddenbar-memory-soak`. Launchd stderr goes to `~/Library/Logs/hiddenbar-memory-soak.err`. +## Heartbeat + +Each completed sample pings the `mini.hiddenbar-memory-soak` vps-mon heartbeat (`~/.config/vps-mon/hiddenbar-memory-soak-heartbeat-url`, through vps-mon `bin/hb-ping`). A sample with no footprint reading does not ping. No URL file, or a failed ping, changes nothing. The end-date branch does not ping, so the heartbeat goes silent once the soak is over: at the end epoch (1796051986, 2026-11-30) retire it with vps-mon `bin/retire-heartbeat mini.hiddenbar-memory-soak`. `scripts/memory-soak/test-heartbeat` proves it. + ## Reading a leak A leak is footprint and node count both rising across days at the same pid; a new pid (app restarted) restarts the series, so compare within a pid only. diff --git a/scripts/memory-soak/sample b/scripts/memory-soak/sample index fba4a91e..5057c99e 100755 --- a/scripts/memory-soak/sample +++ b/scripts/memory-soak/sample @@ -8,7 +8,7 @@ # Env: HB_SOAK_LOG (default ~/Library/Logs/hiddenbar-memory-soak.log), HB_SOAK_APP, HB_SOAK_LABEL, # HB_SOAK_HB_FILE and HB_SOAK_PING (the vps-mon heartbeat URL file and the hb-ping helper). # -# Liveness: each completed sample (app running or not) pings the mini.hiddenbar-memory-soak heartbeat. +# Liveness: each completed sample (app running or not, but not one with footprint=NA) pings the mini.hiddenbar-memory-soak heartbeat. # No URL file or a failed ping never changes the exit code. The end-epoch branch does NOT ping, so the # heartbeat goes silent after the soak by design: retire it with vps-mon bin/retire-heartbeat then. # Line: TS pid=N up=ETIME footprint=B peak=B nodes=N heap_bytes=B version=SHORT+BUILD @@ -72,4 +72,5 @@ if [ -z "$nodes" ]; then fi log "pid=$pid up=$up footprint=$footprint peak=$peak nodes=$nodes heap_bytes=$heap_bytes version=$version$note" -heartbeat +# A sample without a footprint reading did not measure anything: stay silent so a broken sampler alerts. +[ "$footprint" = NA ] || heartbeat diff --git a/scripts/memory-soak/test-heartbeat b/scripts/memory-soak/test-heartbeat index bcad619c..76990ae2 100755 --- a/scripts/memory-soak/test-heartbeat +++ b/scripts/memory-soak/test-heartbeat @@ -7,6 +7,8 @@ # no-url URL file absent -> 0 pings, exit 0 # soak-ended END_EPOCH already past (NEGATIVE CONTROL) -> 0 pings, exit 0 # ping-fails ping helper exits non-zero -> exit 0, "heartbeat ping failed" logged +# running a fake "Hidden Bar" process, real footprint -> 1 ping (the sampled-and-measured path) +# no-footprint same process, `footprint` stubbed to print nothing -> 0 pings (footprint=NA stays silent) # Exit 0 when every case matches, 1 otherwise. set -u @@ -20,12 +22,24 @@ printf '#!/bin/bash\nexit 7\n' > "$T/ping-bad" chmod +x "$T/ping-ok" "$T/ping-bad" echo "https://example.invalid/v1/heartbeat/FIXTURE" > "$T/url" +# A fake app: a process whose command line contains "/Contents/MacOS/Hidden Bar", which is what sample pgreps. +FAKE="$T/Fake Hidden Bar.app" +mkdir -p "$FAKE/Contents/MacOS" "$T/stubbin" +printf '#!/bin/bash\nsleep 120\n' > "$FAKE/Contents/MacOS/Hidden Bar" +chmod +x "$FAKE/Contents/MacOS/Hidden Bar" +"$FAKE/Contents/MacOS/Hidden Bar" & +FAKEPID=$! +trap 'kill $FAKEPID 2>/dev/null' EXIT +printf '#!/bin/bash\nexit 0\n' > "$T/stubbin/footprint" +chmod +x "$T/stubbin/footprint" +sleep 1 + # run run() { : >| "$T/hits" : >| "$T/log" - HB_SOAK_LOG="$T/log" HB_SOAK_APP="$T/NoSuch.app" HB_SOAK_LABEL=mini.hiddenbar-memory-soak-test \ - HB_SOAK_HB_FILE="$3" HB_SOAK_PING="$2" "$SAMPLE" $4 >/dev/null 2>&1 + HB_SOAK_LOG="$T/log" HB_SOAK_APP="${APP:-$T/NoSuch.app}" HB_SOAK_LABEL=mini.hiddenbar-memory-soak-test \ + HB_SOAK_HB_FILE="$3" HB_SOAK_PING="$2" env PATH="${EXTRA_PATH:+$EXTRA_PATH:}$PATH" "$SAMPLE" $4 >/dev/null 2>&1 local rc=$? pings pings=$(wc -l < "$T/hits" | tr -d ' ') if [ "$rc" = 0 ] && [ "$pings" = "$5" ] && { [ -z "$6" ] || grep -q "$6" "$T/log"; }; then @@ -40,6 +54,8 @@ run sampled "$T/ping-ok" "$T/url" "" 1 "state=not-running" run no-url "$T/ping-ok" "$T/none" "" 0 "state=not-running" run soak-ended "$T/ping-ok" "$T/url" 1 0 "soak-ended" run ping-fails "$T/ping-bad" "$T/url" "" 0 "heartbeat ping failed" +APP="$FAKE" run running "$T/ping-ok" "$T/url" "" 1 "footprint=[0-9]" +APP="$FAKE" EXTRA_PATH="$T/stubbin" run no-footprint "$T/ping-ok" "$T/url" "" 0 "footprint=NA" [ "$fails" = 0 ] && echo "-- summary: all cases match" || echo "-- summary: $fails case(s) FAIL" [ "$fails" = 0 ]