From b96c4f542b3011f5d67d181130a9fd490940a53b Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Sun, 7 Jun 2026 16:04:35 +0300 Subject: [PATCH 1/2] fix(anvil): survive OOM-truncated state + bound memory growth The internal anvil devnet (rpc-dev) crash-looped ~4900 times until Caddy served only 502s. Root cause: with a fixed block-time anvil's in-memory chain history grows without bound; on the ~3.8 GB / no-swap VPS it reached ~3.7 GB and the global OOM-killer killed it mid state-snapshot, truncating /var/lib/anvil/state.json to 0 bytes. anvil's --state then refuses to parse the empty file ("EOF while parsing a value at line 1 column 0", exit 2), wedging the unit permanently. Three layered fixes in the anvil role: - anvil-state-guard.sh as ExecStartPre: quarantines an empty/corrupt state.json (moves it aside) so a truncated snapshot self-heals to a fresh chain instead of crash-looping. A devnet can lose state; it can't be down. - --prune-history (default 5000) bounds in-memory history so the box can't OOM in the first place. --transaction-block-keeper is exposed too (off). - Optional cgroup MemoryHigh/MemoryMax in the unit, set for the rpc-dev VPS in group_vars (2G/2560M), so a runaway is confined to its own slice rather than letting the global OOM-killer pick Caddy or sshd. Verified: template renders to valid systemd for flags on/off; ansible-lint (production), yamllint, shellcheck, KICS (0 high/critical) all pass; molecule converge+verify boots the new unit (anvil up with --prune-history, chain-id 0x7a69, loopback-only, 401 gate intact). Co-Authored-By: Claude Opus 4.8 (1M context) --- ansible/inventory/group_vars/anvil_devnet.yml | 9 ++++++ ansible/roles/anvil/defaults/main.yml | 19 +++++++++++ ansible/roles/anvil/tasks/main.yml | 13 ++++++++ .../anvil/templates/anvil-state-guard.sh.j2 | 32 +++++++++++++++++++ .../roles/anvil/templates/anvil.service.j2 | 28 ++++++++++++++++ 5 files changed, 101 insertions(+) create mode 100644 ansible/roles/anvil/templates/anvil-state-guard.sh.j2 diff --git a/ansible/inventory/group_vars/anvil_devnet.yml b/ansible/inventory/group_vars/anvil_devnet.yml index c5837f4..0aafea5 100644 --- a/ansible/inventory/group_vars/anvil_devnet.yml +++ b/ansible/inventory/group_vars/anvil_devnet.yml @@ -15,3 +15,12 @@ baseline_extra_inbound: # Optional Let's Encrypt account contact (used by the caddy role's global block): # caddy_acme_email: ops@decdn.org + +# --- anvil memory ceilings (this host is a ~3.8 GB / no-swap VPS) -------------- +# Sized to leave headroom for Caddy + the system on a small box. On 2026-06-07 an +# unbounded anvil grew to ~3.7 GB and the GLOBAL OOM-killer killed it mid state +# snapshot, truncating state.json and crash-looping the unit. The cgroup cap below +# (plus anvil_prune_history in the role defaults) confines a runaway to its own +# slice; raise both on a larger host. +anvil_memory_high: "2G" +anvil_memory_max: "2560M" diff --git a/ansible/roles/anvil/defaults/main.yml b/ansible/roles/anvil/defaults/main.yml index 5de12f9..833426f 100644 --- a/ansible/roles/anvil/defaults/main.yml +++ b/ansible/roles/anvil/defaults/main.yml @@ -8,6 +8,25 @@ anvil_accounts: 10 anvil_host: "127.0.0.1" # loopback ONLY — never 0.0.0.0 (hard rule #2) anvil_port: 8545 +# --- Resilience: bound in-memory growth so the box can't OOM the daemon -------- +# anvil keeps chain history in RAM; with a fixed block-time it grows without bound +# and eventually trips the kernel OOM-killer (which can truncate the state file and +# wedge the service — see anvil-state-guard.sh). prune-history caps the number of +# historical states kept in memory. Set falsy to keep full history (the old, OOM- +# prone behaviour). transaction-block-keeper is optional and omitted unless set. +anvil_prune_history: 5000 # --prune-history N (falsy => omit, keep full history) +anvil_transaction_block_keeper: "" # --transaction-block-keeper N ("" => omit) + +# cgroup memory ceilings for the unit. Empty => no limit (role default). Sizing is +# host-specific, so set these in inventory/group_vars for small VPSes — a cgroup cap +# confines a runaway anvil to its own slice instead of letting the GLOBAL OOM-killer +# pick a victim (e.g. Caddy or sshd). MemoryHigh throttles via reclaim; MemoryMax is +# the hard kill. Pair with the state guard so a MemoryMax kill self-heals on restart. +anvil_memory_high: "" # e.g. "2G" (systemd MemoryHigh=, "" => omit) +anvil_memory_max: "" # e.g. "2560M" (systemd MemoryMax=, "" => omit) + +anvil_state_guard_bin: /usr/local/sbin/anvil-state-guard + # --- Foundry --- foundry_dir: /opt/foundry foundry_version: latest # foundryup -i value: latest | nightly | vX.Y.Z (pin for reproducibility) diff --git a/ansible/roles/anvil/tasks/main.yml b/ansible/roles/anvil/tasks/main.yml index 2edbacd..2e4103d 100644 --- a/ansible/roles/anvil/tasks/main.yml +++ b/ansible/roles/anvil/tasks/main.yml @@ -138,6 +138,19 @@ {{ anvil_effective_mnemonic }} when: anvil_mnemonic_supplied is not defined or anvil_mnemonic_supplied | length == 0 +# --- State-resilience guard (ExecStartPre) ------------------------------------ +# Quarantines an empty/corrupt state.json so an OOM-truncated snapshot can't wedge +# the unit in a crash loop. Installed before the unit so the ExecStartPre target +# exists the first time the unit (re)starts. +- name: Install the anvil state-resilience guard + ansible.builtin.template: + src: anvil-state-guard.sh.j2 + dest: "{{ anvil_state_guard_bin }}" + owner: root + group: root + mode: "0755" + notify: Restart anvil + # --- Hardened systemd unit ---------------------------------------------------- - name: Install anvil systemd unit ansible.builtin.template: diff --git a/ansible/roles/anvil/templates/anvil-state-guard.sh.j2 b/ansible/roles/anvil/templates/anvil-state-guard.sh.j2 new file mode 100644 index 0000000..1d5acca --- /dev/null +++ b/ansible/roles/anvil/templates/anvil-state-guard.sh.j2 @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# MANAGED BY the anvil role — do not edit by hand. +# +# anvil's --state flag refuses to start when the snapshot file exists but is empty +# or not valid JSON (it aborts with "EOF while parsing a value at line 1 column 0", +# exit 2). An OOM-kill mid-snapshot (--state-interval writes) truncates the file to +# zero bytes, which then wedges the service into a permanent restart loop — exactly +# the incident this guard exists to prevent. +# +# Run as ExecStartPre: if the state file is empty or corrupt, move it aside so anvil +# starts from a fresh chain instead of crash-looping. A devnet can afford to lose +# state; it cannot afford to be down. The moved-aside file is kept for forensics. +set -euo pipefail + +state="{{ anvil_state_dir }}/state.json" + +# No snapshot yet (first boot): anvil will create one. Nothing to guard. +[ -e "$state" ] || exit 0 + +bad=0 +if [ ! -s "$state" ]; then + bad=1 # zero bytes — the OOM-truncation failure mode +elif command -v jq >/dev/null 2>&1 && ! jq -e . "$state" >/dev/null 2>&1; then + bad=1 # non-empty but not parseable JSON (partial write) +fi + +if [ "$bad" -eq 1 ]; then + ts="$(date -u +%Y%m%dT%H%M%SZ)" + corrupt="${state}.corrupt-${ts}" + echo "anvil-state-guard: '${state}' is empty or invalid JSON; moving to '${corrupt}' so anvil can start from a fresh chain" >&2 + mv -f "$state" "$corrupt" +fi diff --git a/ansible/roles/anvil/templates/anvil.service.j2 b/ansible/roles/anvil/templates/anvil.service.j2 index ae79883..14d3de2 100644 --- a/ansible/roles/anvil/templates/anvil.service.j2 +++ b/ansible/roles/anvil/templates/anvil.service.j2 @@ -4,6 +4,11 @@ # {{ anvil_host }}:{{ anvil_port }} ONLY (never 0.0.0.0), and persists chain state # to {{ anvil_state_dir }}/state.json (loaded on boot, dumped on SIGTERM, # snapshotted every 30s so an unclean reboot loses at most that interval). +# +# Resilience: ExecStartPre runs anvil-state-guard to quarantine an empty/corrupt +# snapshot (an OOM-truncated state.json otherwise wedges the unit in a crash loop), +# --prune-history bounds in-memory growth, and the cgroup Memory* limits below cap +# the process so a runaway can't take the whole box down via the global OOM-killer. [Unit] Description=deCDN anvil devnet (EVM settlement layer) Documentation=https://github.com/decdn/decdn-devops @@ -24,6 +29,10 @@ EnvironmentFile={{ anvil_env_file }} # phrase contains spaces. Caveat: it is visible in `ps`/`systemctl show` ON THIS # HOST — acceptable because it only controls funded *test* accounts, and is why # the RPC must never be exposed without auth. Do not reuse it with real value. +# Quarantine an empty/corrupt snapshot before anvil reads it, so a truncated +# state.json (e.g. from an OOM-kill mid-write) self-heals instead of crash-looping. +ExecStartPre={{ anvil_state_guard_bin }} + ExecStart={{ foundry_dir }}/bin/anvil \ --host {{ anvil_host }} \ --port {{ anvil_port }} \ @@ -31,6 +40,12 @@ ExecStart={{ foundry_dir }}/bin/anvil \ --block-time {{ anvil_block_time }} \ --accounts {{ anvil_accounts }} \ --mnemonic ${ANVIL_MNEMONIC} \ +{% if anvil_prune_history %} + --prune-history {{ anvil_prune_history }} \ +{% endif %} +{% if anvil_transaction_block_keeper %} + --transaction-block-keeper {{ anvil_transaction_block_keeper }} \ +{% endif %} --state {{ anvil_state_dir }}/state.json \ --state-interval 30 @@ -41,6 +56,19 @@ TimeoutStopSec=30 Restart=always RestartSec=2 +# --- Memory ceilings (cgroup) ---------------------------------------------- +# Confine a runaway anvil to its own slice: MemoryHigh throttles via reclaim, +# MemoryMax is the hard kill. Without these, unbounded history growth invokes the +# GLOBAL OOM-killer, which may pick Caddy/sshd instead. A MemoryMax kill truncates +# the snapshot the same way — the state guard above makes that recoverable. Sizing +# is host-specific; set in inventory/group_vars (empty here => no limit). +{% if anvil_memory_high %} +MemoryHigh={{ anvil_memory_high }} +{% endif %} +{% if anvil_memory_max %} +MemoryMax={{ anvil_memory_max }} +{% endif %} + # --- Hardening ------------------------------------------------------------- # StateDirectory creates/owns {{ anvil_state_dir }} (0750) and is the only # writable path the process gets under ProtectSystem=strict. From 6d86cbbe8ad3dd913a41f786086d3a7a9c168ff1 Mon Sep 17 00:00:00 2001 From: Ant Somers Date: Sun, 7 Jun 2026 16:13:34 +0300 Subject: [PATCH 2/2] fix(anvil): make state guard O(1) memory + bounded disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review on the ExecStartPre guard. It runs inside the unit's cgroup, so it is subject to the same MemoryMax as anvil — validating a large snapshot with jq could itself be OOM-killed and re-wedge startup, the exact failure the guard exists to prevent. Replace the jq parse with an O(1) check: file is non-empty and its last non-whitespace byte is a JSON close token (} or ]); a truncated write won't end that way. This also drops the jq dependency. Quarantine to a static '.corrupt' name instead of a timestamped one so a repeated crash/OOM loop overwrites a single file rather than accumulating snapshots and exhausting disk on the constrained VPS; the latest sample still suffices for forensics. Drops the date dependency too. Verified: shellcheck clean; behavioural test covers no-file/empty/truncated/ valid (valid snapshot preserved, empty and truncated quarantined). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../anvil/templates/anvil-state-guard.sh.j2 | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/ansible/roles/anvil/templates/anvil-state-guard.sh.j2 b/ansible/roles/anvil/templates/anvil-state-guard.sh.j2 index 1d5acca..6a4fab5 100644 --- a/ansible/roles/anvil/templates/anvil-state-guard.sh.j2 +++ b/ansible/roles/anvil/templates/anvil-state-guard.sh.j2 @@ -2,14 +2,20 @@ # MANAGED BY the anvil role — do not edit by hand. # # anvil's --state flag refuses to start when the snapshot file exists but is empty -# or not valid JSON (it aborts with "EOF while parsing a value at line 1 column 0", -# exit 2). An OOM-kill mid-snapshot (--state-interval writes) truncates the file to -# zero bytes, which then wedges the service into a permanent restart loop — exactly -# the incident this guard exists to prevent. +# or truncated (it aborts with "EOF while parsing a value at line 1 column 0", +# exit 2). An OOM-kill mid-snapshot (--state-interval writes) leaves exactly such a +# file, which then wedges the service into a permanent restart loop — the incident +# this guard exists to prevent. # -# Run as ExecStartPre: if the state file is empty or corrupt, move it aside so anvil -# starts from a fresh chain instead of crash-looping. A devnet can afford to lose -# state; it cannot afford to be down. The moved-aside file is kept for forensics. +# Run as ExecStartPre: if the state file is empty or doesn't end in a JSON close +# token, move it aside so anvil starts from a fresh chain instead of crash-looping. +# A devnet can afford to lose state; it cannot afford to be down. +# +# The checks are deliberately O(1) in memory. This runs INSIDE the unit's cgroup, +# so it is subject to the same MemoryMax as anvil — parsing a potentially large +# snapshot with jq could itself be OOM-killed and re-wedge startup, the very thing +# we are guarding against. A size test plus a trailing-token test catches the empty- +# and truncated-write failure modes without loading the file into memory. set -euo pipefail state="{{ anvil_state_dir }}/state.json" @@ -20,13 +26,20 @@ state="{{ anvil_state_dir }}/state.json" bad=0 if [ ! -s "$state" ]; then bad=1 # zero bytes — the OOM-truncation failure mode -elif command -v jq >/dev/null 2>&1 && ! jq -e . "$state" >/dev/null 2>&1; then - bad=1 # non-empty but not parseable JSON (partial write) +else + # A complete anvil snapshot is a JSON document, so its last non-whitespace byte + # is a close token. A write truncated mid-content won't end this way. + last="$(tail -c 256 "$state" | tr -d '[:space:]' | tail -c 1)" + if [ "$last" != "}" ] && [ "$last" != "]" ]; then + bad=1 # non-empty but not a complete JSON document (partial write) + fi fi if [ "$bad" -eq 1 ]; then - ts="$(date -u +%Y%m%dT%H%M%SZ)" - corrupt="${state}.corrupt-${ts}" - echo "anvil-state-guard: '${state}' is empty or invalid JSON; moving to '${corrupt}' so anvil can start from a fresh chain" >&2 + # Static name (not timestamped): a repeated crash/OOM loop overwrites one file + # instead of filling the disk with snapshots. The latest sample suffices for + # forensics, and the disk headroom matters more on this constrained host. + corrupt="${state}.corrupt" + echo "anvil-state-guard: '${state}' is empty or truncated; moving to '${corrupt}' so anvil can start from a fresh chain" >&2 mv -f "$state" "$corrupt" fi