From 512f58aaa1d3f9ce71ed9154b74dce972ec171d2 Mon Sep 17 00:00:00 2001 From: Nader Helmy Date: Wed, 29 Jul 2026 04:28:27 -0500 Subject: [PATCH] fix: give log-node and archive-node real health-check margin All four services already had Fly health checks and all four were passing, so this is not about adding coverage. It is about two grace periods that were set below, or barely above, how long the service actually takes to start. Measured 2026-07-29 from the Fly logs, "Starting init" to "listening": log-node 5s startup, 5s grace no margin archive-node 4s startup, 5s grace 1s directory-node 3s grace 15s 5x, left alone grace_period is the window before a failing check counts, and neither service binds its port until startup finishes. At zero margin any slower boot marks a healthy service unhealthy. Both numbers also grow. log-node replays the persisted log into its in-memory Merkle tree at startup, so it scales with tree_size, ~115k entries replaying at roughly 23k/sec on the measurement date. archive-node replays archived record bodies, currently 13, which is why its tight value has not bitten yet. log-node goes to 45s and archive-node to 30s, roughly 9x and 7x measured boot. directory-node keeps 15s and gains the measurement in a comment so the next person can re-check rather than re-derive. Every file now records the ceiling: Fly clamps grace_period at 60s and lowers anything larger without failing, so past that the fix is a cheaper startup rather than a longer grace. That clamp is the same constraint P059 records for graph-node, where archive replay time is what bounds machine sizing. --- services/archive-node/fly.toml | 12 +++++++++++- services/directory-node/fly.toml | 8 ++++++++ services/log-node/fly.toml | 17 ++++++++++++++++- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/services/archive-node/fly.toml b/services/archive-node/fly.toml index 57c63187c..d101f772e 100644 --- a/services/archive-node/fly.toml +++ b/services/archive-node/fly.toml @@ -32,6 +32,16 @@ primary_region = 'iad' type = 'http' interval = '15s' timeout = '2s' - grace_period = '5s' + # grace_period MUST exceed boot-to-listening. Measured 2026-07-29 from the + # Fly logs: "Starting init" 08:57:14 -> "atrib-archive listening" 08:57:18, + # so 4 seconds against a 5 second grace. One second of margin is not margin. + # + # Startup replays the archived record bodies ("replayed 13 archived + # records"), so it grows with opt-in submissions. 13 is small today, which + # is exactly why the tight value has not bitten yet. + # + # 30s is ~7x the measured boot. Fly clamps this field at 60s, so that is the + # ceiling; past it the fix is a cheaper startup, not a longer grace. + grace_period = '30s' method = 'GET' path = '/v1/retention' diff --git a/services/directory-node/fly.toml b/services/directory-node/fly.toml index e922a88ab..df6c0276e 100644 --- a/services/directory-node/fly.toml +++ b/services/directory-node/fly.toml @@ -40,4 +40,12 @@ primary_region = 'iad' path = '/v6/anchor' interval = '30s' timeout = '5s' + # Left at 15s: measured 2026-07-29, "Starting init" 08:56:59 -> "directory-node + # listening" 08:57:02, so 3 seconds with 5x margin. Unlike log-node and + # archive-node, which were sitting at zero and one second of margin and were + # raised in the same change, this one has room. + # + # It is not fixed forever. Startup replays persisted claims into a fresh + # in-memory AKD ("replayed 20 persisted claims"), so re-measure if claim + # volume grows by an order of magnitude. Fly clamps this field at 60s. grace_period = '15s' diff --git a/services/log-node/fly.toml b/services/log-node/fly.toml index 92ab03c32..2f8948493 100644 --- a/services/log-node/fly.toml +++ b/services/log-node/fly.toml @@ -38,7 +38,22 @@ primary_region = 'iad' type = 'http' interval = '15s' timeout = '2s' - grace_period = '5s' + # grace_period MUST exceed boot-to-listening. Measured 2026-07-29 from the + # Fly logs: "Starting init" 08:57:02 -> "atrib-log listening" 08:57:07, so + # 5 seconds against a 5 second grace, i.e. no margin at all. Any slower + # boot fails the check on a service that is merely starting. + # + # It also grows: as the comment on [[vm]] memory below says, the persisted + # log is replayed into the in-memory Merkle tree at startup, so this scales + # with tree_size (~115k entries on the measurement date, replaying at + # roughly 23k entries/sec). + # + # 45s is ~9x the measured boot and covers on the order of a million entries + # before it needs revisiting. Fly clamps this field at 60s (`flyctl config + # validate` warns "this will be lowered to 1 minute"), so 60s is the hard + # ceiling, not a value to reach for casually: past it the only fix is + # making startup cheaper. + grace_period = '45s' method = 'GET' # /v1/pubkey always returns 200 (even with an empty tree); /v1/checkpoint # returns 404 until the first entry lands. With persistence in place a