fix(anvil): survive OOM-truncated state + bound memory growth

ansible/inventory/group_vars/anvil_devnet.yml

@@ -15,3 +15,12 @@ baseline_extra_inbound:
 
 # Optional Let's Encrypt account contact (used by the caddy role's global block):
 # caddy_acme_email: ops@decdn.org
+
+# --- anvil memory ceilings (this host is a ~3.8 GB / no-swap VPS) --------------
+# Sized to leave headroom for Caddy + the system on a small box. On 2026-06-07 an
+# unbounded anvil grew to ~3.7 GB and the GLOBAL OOM-killer killed it mid state
+# snapshot, truncating state.json and crash-looping the unit. The cgroup cap below
+# (plus anvil_prune_history in the role defaults) confines a runaway to its own
+# slice; raise both on a larger host.
+anvil_memory_high: "2G"
+anvil_memory_max: "2560M"

ansible/roles/anvil/defaults/main.yml

@@ -8,6 +8,25 @@ anvil_accounts: 10
 anvil_host: "127.0.0.1"          # loopback ONLY — never 0.0.0.0 (hard rule #2)
 anvil_port: 8545
 
+# --- Resilience: bound in-memory growth so the box can't OOM the daemon --------
+# anvil keeps chain history in RAM; with a fixed block-time it grows without bound
+# and eventually trips the kernel OOM-killer (which can truncate the state file and
+# wedge the service — see anvil-state-guard.sh). prune-history caps the number of
+# historical states kept in memory. Set falsy to keep full history (the old, OOM-
+# prone behaviour). transaction-block-keeper is optional and omitted unless set.
+anvil_prune_history: 5000            # --prune-history N  (falsy => omit, keep full history)
+anvil_transaction_block_keeper: ""   # --transaction-block-keeper N  ("" => omit)
+
+# cgroup memory ceilings for the unit. Empty => no limit (role default). Sizing is
+# host-specific, so set these in inventory/group_vars for small VPSes — a cgroup cap
+# confines a runaway anvil to its own slice instead of letting the GLOBAL OOM-killer
+# pick a victim (e.g. Caddy or sshd). MemoryHigh throttles via reclaim; MemoryMax is
+# the hard kill. Pair with the state guard so a MemoryMax kill self-heals on restart.
+anvil_memory_high: ""                # e.g. "2G"     (systemd MemoryHigh=, "" => omit)
+anvil_memory_max: ""                 # e.g. "2560M"  (systemd MemoryMax=, "" => omit)
+
+anvil_state_guard_bin: /usr/local/sbin/anvil-state-guard
+
 # --- Foundry ---
 foundry_dir: /opt/foundry
 foundry_version: latest          # foundryup -i value: latest | nightly | vX.Y.Z (pin for reproducibility)

ansible/roles/anvil/tasks/main.yml

@@ -138,6 +138,19 @@
           {{ anvil_effective_mnemonic }}
       when: anvil_mnemonic_supplied is not defined or anvil_mnemonic_supplied | length == 0
 
+# --- State-resilience guard (ExecStartPre) ------------------------------------
+# Quarantines an empty/corrupt state.json so an OOM-truncated snapshot can't wedge
+# the unit in a crash loop. Installed before the unit so the ExecStartPre target
+# exists the first time the unit (re)starts.
+- name: Install the anvil state-resilience guard
+  ansible.builtin.template:
+    src: anvil-state-guard.sh.j2
+    dest: "{{ anvil_state_guard_bin }}"
+    owner: root
+    group: root
+    mode: "0755"
+  notify: Restart anvil
+
 # --- Hardened systemd unit ----------------------------------------------------
 - name: Install anvil systemd unit
   ansible.builtin.template:
@@ -163,7 +176,7 @@
 
 - name: Wait for the anvil RPC to answer on loopback
   ansible.builtin.uri:
     url: "http://{{ anvil_host }}:{{ anvil_port }}"
     method: POST
     body_format: json
     body:

ansible/roles/anvil/templates/anvil-state-guard.sh.j2

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# MANAGED BY the anvil role — do not edit by hand.
+#
+# anvil's --state flag refuses to start when the snapshot file exists but is empty
+# or truncated (it aborts with "EOF while parsing a value at line 1 column 0",
+# exit 2). An OOM-kill mid-snapshot (--state-interval writes) leaves exactly such a
+# file, which then wedges the service into a permanent restart loop — the incident
+# this guard exists to prevent.
+#
+# Run as ExecStartPre: if the state file is empty or doesn't end in a JSON close
+# token, move it aside so anvil starts from a fresh chain instead of crash-looping.
+# A devnet can afford to lose state; it cannot afford to be down.
+#
+# The checks are deliberately O(1) in memory. This runs INSIDE the unit's cgroup,
+# so it is subject to the same MemoryMax as anvil — parsing a potentially large
+# snapshot with jq could itself be OOM-killed and re-wedge startup, the very thing
+# we are guarding against. A size test plus a trailing-token test catches the empty-
+# and truncated-write failure modes without loading the file into memory.
+set -euo pipefail
+
+state="{{ anvil_state_dir }}/state.json"
+
+# No snapshot yet (first boot): anvil will create one. Nothing to guard.
+[ -e "$state" ] || exit 0
+
+bad=0
+if [ ! -s "$state" ]; then
+    bad=1   # zero bytes — the OOM-truncation failure mode
+else
+    # A complete anvil snapshot is a JSON document, so its last non-whitespace byte
+    # is a close token. A write truncated mid-content won't end this way.
+    last="$(tail -c 256 "$state" | tr -d '[:space:]' | tail -c 1)"
+    if [ "$last" != "}" ] && [ "$last" != "]" ]; then
+        bad=1   # non-empty but not a complete JSON document (partial write)
+    fi
+fi
+
+if [ "$bad" -eq 1 ]; then
+    # Static name (not timestamped): a repeated crash/OOM loop overwrites one file
+    # instead of filling the disk with snapshots. The latest sample suffices for
+    # forensics, and the disk headroom matters more on this constrained host.
+    corrupt="${state}.corrupt"
+    echo "anvil-state-guard: '${state}' is empty or truncated; moving to '${corrupt}' so anvil can start from a fresh chain" >&2
+    mv -f "$state" "$corrupt"
+fi

ansible/roles/anvil/templates/anvil.service.j2

@@ -4,6 +4,11 @@
 # {{ anvil_host }}:{{ anvil_port }} ONLY (never 0.0.0.0), and persists chain state
 # to {{ anvil_state_dir }}/state.json (loaded on boot, dumped on SIGTERM,
 # snapshotted every 30s so an unclean reboot loses at most that interval).
+#
+# Resilience: ExecStartPre runs anvil-state-guard to quarantine an empty/corrupt
+# snapshot (an OOM-truncated state.json otherwise wedges the unit in a crash loop),
+# --prune-history bounds in-memory growth, and the cgroup Memory* limits below cap
+# the process so a runaway can't take the whole box down via the global OOM-killer.
 [Unit]
 Description=deCDN anvil devnet (EVM settlement layer)
 Documentation=https://github.com/decdn/decdn-devops
@@ -24,13 +29,23 @@ EnvironmentFile={{ anvil_env_file }}
 # phrase contains spaces. Caveat: it is visible in `ps`/`systemctl show` ON THIS
 # HOST — acceptable because it only controls funded *test* accounts, and is why
 # the RPC must never be exposed without auth. Do not reuse it with real value.
+# Quarantine an empty/corrupt snapshot before anvil reads it, so a truncated
+# state.json (e.g. from an OOM-kill mid-write) self-heals instead of crash-looping.
+ExecStartPre={{ anvil_state_guard_bin }}
+
 ExecStart={{ foundry_dir }}/bin/anvil \
     --host {{ anvil_host }} \
     --port {{ anvil_port }} \
     --chain-id {{ anvil_chain_id }} \
     --block-time {{ anvil_block_time }} \
     --accounts {{ anvil_accounts }} \
     --mnemonic ${ANVIL_MNEMONIC} \
+{% if anvil_prune_history %}
+    --prune-history {{ anvil_prune_history }} \
+{% endif %}
+{% if anvil_transaction_block_keeper %}
+    --transaction-block-keeper {{ anvil_transaction_block_keeper }} \
+{% endif %}
     --state {{ anvil_state_dir }}/state.json \
     --state-interval 30
 
@@ -41,6 +56,19 @@ TimeoutStopSec=30
 Restart=always
 RestartSec=2
 
+# --- Memory ceilings (cgroup) ----------------------------------------------
+# Confine a runaway anvil to its own slice: MemoryHigh throttles via reclaim,
+# MemoryMax is the hard kill. Without these, unbounded history growth invokes the
+# GLOBAL OOM-killer, which may pick Caddy/sshd instead. A MemoryMax kill truncates
+# the snapshot the same way — the state guard above makes that recoverable. Sizing
+# is host-specific; set in inventory/group_vars (empty here => no limit).
+{% if anvil_memory_high %}
+MemoryHigh={{ anvil_memory_high }}
+{% endif %}
+{% if anvil_memory_max %}
+MemoryMax={{ anvil_memory_max }}
+{% endif %}
+
 # --- Hardening -------------------------------------------------------------
 # StateDirectory creates/owns {{ anvil_state_dir }} (0750) and is the only
 # writable path the process gets under ProtectSystem=strict.