diff --git a/services/archive-node/fly.toml b/services/archive-node/fly.toml index 57c63187..d101f772 100644 --- a/services/archive-node/fly.toml +++ b/services/archive-node/fly.toml @@ -32,6 +32,16 @@ primary_region = 'iad' type = 'http' interval = '15s' timeout = '2s' - grace_period = '5s' + # grace_period MUST exceed boot-to-listening. Measured 2026-07-29 from the + # Fly logs: "Starting init" 08:57:14 -> "atrib-archive listening" 08:57:18, + # so 4 seconds against a 5 second grace. One second of margin is not margin. + # + # Startup replays the archived record bodies ("replayed 13 archived + # records"), so it grows with opt-in submissions. 13 is small today, which + # is exactly why the tight value has not bitten yet. + # + # 30s is ~7x the measured boot. Fly clamps this field at 60s, so that is the + # ceiling; past it the fix is a cheaper startup, not a longer grace. + grace_period = '30s' method = 'GET' path = '/v1/retention' diff --git a/services/directory-node/fly.toml b/services/directory-node/fly.toml index e922a88a..df6c0276 100644 --- a/services/directory-node/fly.toml +++ b/services/directory-node/fly.toml @@ -40,4 +40,12 @@ primary_region = 'iad' path = '/v6/anchor' interval = '30s' timeout = '5s' + # Left at 15s: measured 2026-07-29, "Starting init" 08:56:59 -> "directory-node + # listening" 08:57:02, so 3 seconds with 5x margin. Unlike log-node and + # archive-node, which were sitting at zero and one second of margin and were + # raised in the same change, this one has room. + # + # It is not fixed forever. Startup replays persisted claims into a fresh + # in-memory AKD ("replayed 20 persisted claims"), so re-measure if claim + # volume grows by an order of magnitude. Fly clamps this field at 60s. grace_period = '15s' diff --git a/services/log-node/fly.toml b/services/log-node/fly.toml index 92ab03c3..2f894849 100644 --- a/services/log-node/fly.toml +++ b/services/log-node/fly.toml @@ -38,7 +38,22 @@ primary_region = 'iad' type = 'http' interval = '15s' timeout = '2s' - grace_period = '5s' + # grace_period MUST exceed boot-to-listening. Measured 2026-07-29 from the + # Fly logs: "Starting init" 08:57:02 -> "atrib-log listening" 08:57:07, so + # 5 seconds against a 5 second grace, i.e. no margin at all. Any slower + # boot fails the check on a service that is merely starting. + # + # It also grows: as the comment on [[vm]] memory below says, the persisted + # log is replayed into the in-memory Merkle tree at startup, so this scales + # with tree_size (~115k entries on the measurement date, replaying at + # roughly 23k entries/sec). + # + # 45s is ~9x the measured boot and covers on the order of a million entries + # before it needs revisiting. Fly clamps this field at 60s (`flyctl config + # validate` warns "this will be lowered to 1 minute"), so 60s is the hard + # ceiling, not a value to reach for casually: past it the only fix is + # making startup cheaper. + grace_period = '45s' method = 'GET' # /v1/pubkey always returns 200 (even with an empty tree); /v1/checkpoint # returns 404 until the first entry lands. With persistence in place a