diff --git a/.beads/.gitignore b/.beads/.gitignore new file mode 100644 index 0000000..e72e72e --- /dev/null +++ b/.beads/.gitignore @@ -0,0 +1,46 @@ +# Database +*.db +*.db-journal +*.db-shm +*.db-wal + +# Lock files +*.lock + +# Temporary +last-touched +*.tmp + +# Local history backups +.br_history/ + +# DB-family recovery artifacts (truncated WAL/SHM, quarantined sidecars) +# — same lifecycle as .br_history/, written by recovery paths and +# `br doctor --repair`. Filename suffix `.truncated-wal` slips past the +# generic `*.db-wal` glob above, so it needs an explicit entry (#271). +.br_recovery/ + +# Sync state (local-only, per-machine) +.sync.lock +sync_base.jsonl + +# Merge artifacts (temporary files from 3-way merge) +beads.base.jsonl +beads.base.meta.json +beads.left.jsonl +beads.left.meta.json +beads.right.jsonl +beads.right.meta.json + +# Daemon runtime files +daemon.lock +daemon.log +daemon.pid +bd.sock +sync-state.json + +# Worktree redirect file +redirect + +# bv (beads viewer) lock file +.bv.lock diff --git a/.beads/config.yaml b/.beads/config.yaml new file mode 100644 index 0000000..249aee5 --- /dev/null +++ b/.beads/config.yaml @@ -0,0 +1,4 @@ +# Beads Project Configuration +# issue_prefix: allora-offchain-node +# default_priority: 2 +# default_type: task diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl new file mode 100644 index 0000000..4243bb8 --- /dev/null +++ b/.beads/issues.jsonl @@ -0,0 +1,13 @@ +{"id":"allora-offchain-node-l0o","title":"Port offchain-node chain-client layer to allora-sdk-go","description":"Replace the hand-rolled multi-endpoint / failover / retry / error-classification stack in lib/ with allora-sdk-go (a4f33a6). The sdk now ships both query pools (cosmosrpc.ClientPool, year of prod load) and a tx-send surface (SendTx with code-based retry classification, asg-pvd.1-9) that together cover everything offchain-node currently does badly.\n\nGreg's 5-commit hardening on fix/rpc-silent-stall (65a831c, b32a4c7, 61025f3, 68b2d41, 70204c6) becomes the spec for behaviors the sdk must support: gRPC transport error recognition, CDN-fronted HTTP status parsing, sustained-Idle reaction, fresh-dial on dead endpoint, no monitor goroutine leaks. Each behavior is already inside the sdk surface — this epic verifies and ports onto it.\n\nScope (full migration, both query and tx):\n- DELETE: lib/connection_manager.go, lib/grpcclient/, lib/rpcclient/, lib/mock_connection_manager.go, lib/factory_config.go's chain-client construction, ~half of lib/errors.go (transport/HTTP/grpc-transport classification — now sdk-owned)\n- REPLACE: 12 lib/repo_query_*.go files become one-liner delegations to sdkClient.Cosmos().().(ctx, req)\n- REPLACE: lib/repo_tx_utils.go (SendDataWithRetry, SendTransactionViaRPC) → sdkClient.Tx().SendTx(ctx, signer, msgs, SendOptions{...})\n- KEEP: lib/wallet.go (keyring), lib/types/, actor business logic in usecase/, lib/errors.go ABCI policy triage for app-level errors the sdk's classifyCheckTxError doesn't own (reputer/worker nonce window, read panic, already-submitted, not-permitted)\n- ADAPTER: lib/sdkclient/ (new) — thin ConnectionManagerInterface-shaped adapter over *allora.client so usecase/ layer wiring is minimal\n\nPartition boundaries (for wave-2 parallel implementation, locked after blast-radius scan):\n- Surface A: query repos (12 disjoint files)\n- Surface B: tx path (repo_tx_utils + repo_query_simulate)\n- Surface C: connection manager + grpcclient + rpcclient + factory_config + domain_config (delete/replace)\n- Surface D: errors.go triage scope + usecase wiring + mock\n\nSDK reference snapshot: ~/projects/allora/allora-sdk-go @ a4f33a6 (tani/ENGN-8456-epic-identity-and-wallet-foundation + 25 asg-pvd commits on top of PR #12 tip cec442a). Pre-stage into each worktree at _sdk_reference/ before dispatch.\n\nValidation: go build ./... && go test ./... green. Query-path behavior validated against the 4 CDN-flake patterns greg's tests assert. Tx path validated against ABCI code-based retry (out-of-gas / insufficient-fee / sequence-mismatch).","status":"open","priority":1,"issue_type":"epic","created_at":"2026-06-30T10:19:23.799318Z","created_by":"brynsk","updated_at":"2026-06-30T17:13:28.202847Z","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["epic:sdk-port","reliability"],"comments":[{"id":4,"issue_id":"allora-offchain-node-l0o","author":"brynsk","text":"SDK blockers resolved. Commit e74239e on spook/tx-sending (allora-sdk-go) adds: (1) SendOptions.TimeoutHeight + threading into trySend TxParams, (2) feemarket module via codegen (GasPrice + GasPrices query methods on FeemarketClient, wired into all aggregated clients), (3) PoolParticipant.ResetConnectBackoff() called in probeCooling after failed HealthCheck. go build ./... + go test -short ./... clean. _sdk_reference/ snapshots in all 4 offchain-node worktrees re-synced to e74239e. Wave-2 implementation unblocked for surfaces A, B, C, D.","created_at":"2026-06-30T12:46:57Z"},{"id":5,"issue_id":"allora-offchain-node-l0o","author":"brynsk","text":"Wave-2 implementation + v4-pro review complete. 4 glm-5.2 lanes (sub-aeea3c6d/5b4943c9/416442a2/e7fa3baa) + orchestrator integration merge + v4-pro adversarial review (sub-f1faabbb). Build green, tests green on sdk-port/integration at 2981a91.\n\nv4-pro review caught 6 findings (context was compacted, only Worth Remembering survived; full review body lost to harness compaction — harness bug, not model). Most critical: gas-adjustment double-application (1.2×1.5=1.8 → 50% over-estimate on fees). FIXED in 2981a91: SendOptions.GasAdjustment=1.0, effective multiplier now 1.5.\n\nRemaining v4-pro findings (not blocking, tracked):\n- ErrUnexpectedError sentinel overloaded (exhausted-retries vs unclassified-transport) — retry loop risk. Needs sentinel split.\n- CalculateExponentialBackoffDelaySeconds uses base^retryCount not base×2^retryCount — 81s after 4 retries with base=3. Needs human decision on whether intentional.\n- Pool healthLoop goroutine not stopped on Close() — non-issue for DaemonSet lifecycle.\n- RunWithNodeRetry compat shim error wrapping — needs regression test.\n\nNet diff: -2782/+2230 lines (-552 net). Full hand-rolled stack replaced.","created_at":"2026-06-30T17:13:28Z"}]} +{"id":"allora-offchain-node-l0o.1","title":"Surface A: query-repo migration spec (12 lib/repo_query_*.go)","description":"Produce a migration spec for the 12 lib/repo_query_*.go files. Each currently calls node.Chain.QueryClient.(ctx, req) against a cached *grpc.ClientConn. Target: one-liner delegations to sdkClient.Cosmos().().(ctx, req) via the new lib/sdkclient adapter.\n\nFiles in scope: lib/repo_query_account.go, repo_query_actor_whitelist.go, repo_query_balance.go, repo_query_block.go, repo_query_fee.go, repo_query_nonce.go, repo_query_registration.go, repo_query_simulate.go, repo_query_stake.go, repo_query_status.go, repo_query_topic.go, repo_query_utils.go.\n\nDeliverable (diagnosis only, NO edits): per-file table of (current call signature, sdk equivalent, return-type mapping, test impact). Identify any query method that has NO sdk equivalent (would block the port).","status":"closed","priority":2,"issue_type":"task","created_at":"2026-06-30T10:22:28.469239Z","created_by":"brynsk","updated_at":"2026-06-30T10:43:14.654674Z","closed_at":"2026-06-30T10:43:14.654614Z","close_reason":"Diagnosis complete. 4 glm-5.2 subagents (sub-14c6565d, sub-a6a054b9, sub-57327387, sub-f35616ec) produced migration specs with file:line citations against _sdk_reference/ at a4f33a6. Synthesized into blocker beads l0o.5 (TimeoutHeight), l0o.6 (single-endpoint force-re-dial), l0o.7 (feemarket side-channel) + follow-ups l0o.8 (double gas-adjust), l0o.9 (alerting parity). Wave-2 implementation beads to be filed per-surface once blockers resolve. Partition boundaries locked: A=query repos (12 files), B=tx path (repo_tx_utils+simulate), C=connection layer (delete/replace), D=errors.go scope+usecase wiring (option c: adapter wraps *allora.Client, narrows ConnectionManagerInterface).","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["epic:sdk-port"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.1","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T10:22:28.469239Z","created_by":"brynsk","metadata":"{}","thread_id":""}]} +{"id":"allora-offchain-node-l0o.10","title":"Surface A IMPL: port 12 query repos to sdk","description":"Implement surface A: port the 12 lib/repo_query_*.go files from node.Chain.QueryClient. to adapter.SdkClient().Cosmos().().(ctx, req). Diagnosis spec in l0o.1 comments (closed). All 12 files have exact sdk equivalents — no coverage gaps. feemarket module now in sdk (e74239e). repo_query_utils.go: delete QueryDataWithRetry transport loop (sdk pool owns); retain ProcessErrorTx call only if app-level policy needed (coordinate with surface D). Branch: sdk-port/surface-A-query (worktree pre-staged with _sdk_reference/ at e74239e).","status":"closed","priority":2,"issue_type":"task","created_at":"2026-06-30T13:59:44.889507Z","created_by":"brynsk","updated_at":"2026-06-30T16:51:06.404747Z","closed_at":"2026-06-30T16:51:06.404532Z","close_reason":"Implementation complete. Surfaces A+B+C+D merged into sdk-port/integration (commit 4867948). go build ./... green, go test ./... -short -count=1 green. Net -554 lines (-2782/+2228). Full hand-rolled connection/failover/retry/error-classification stack replaced with allora-sdk-go. 4 glm-5.2 implementation lanes (sub-aeea3c6d, sub-5b4943c9, sub-416442a2, sub-e7fa3baa) + orchestrator-side integration merge.","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["epic:sdk-port","impl"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.10","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T13:59:44.889507Z","created_by":"brynsk","metadata":"{}","thread_id":""}]} +{"id":"allora-offchain-node-l0o.11","title":"Surface B IMPL: port tx path to sdk SendTx","description":"Implement surface B: replace SendDataWithRetry + SendTransactionViaRPC with sdkClient.Tx().SendTx(ctx, signer, msgs, SendOptions{...}). Diagnosis spec in l0o.2 comments. SendOptions.TimeoutHeight now in sdk (e74239e). 5 callsites: 3 registration/stake (Shape A, timeoutHeight=0) + 2 worker/reputer payload (Shape B, timeoutHeight from submission window). Wallet satisfies allora.Signer via GetPrivKey() — no adapter. Gas-price routine feeds SendOptions.GasPrice dynamically. FeeGranter parsed from WalletConfig.FeeGranterAddress. App-level error policy (nonce window, read panic, already-submitted, not-permitted) survives as post-SendTx classifier. Branch: sdk-port/surface-B-tx (worktree pre-staged).","status":"closed","priority":2,"issue_type":"task","created_at":"2026-06-30T13:59:44.982530Z","created_by":"brynsk","updated_at":"2026-06-30T16:51:06.407480Z","closed_at":"2026-06-30T16:51:06.407438Z","close_reason":"Implementation complete. Surfaces A+B+C+D merged into sdk-port/integration (commit 4867948). go build ./... green, go test ./... -short -count=1 green. Net -554 lines (-2782/+2228). Full hand-rolled connection/failover/retry/error-classification stack replaced with allora-sdk-go. 4 glm-5.2 implementation lanes (sub-aeea3c6d, sub-5b4943c9, sub-416442a2, sub-e7fa3baa) + orchestrator-side integration merge.","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["epic:sdk-port","impl"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.11","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T13:59:44.982530Z","created_by":"brynsk","metadata":"{}","thread_id":""}]} +{"id":"allora-offchain-node-l0o.12","title":"Surface C IMPL: delete connection layer, build sdkclient adapter","description":"Implement surface C: DELETE lib/connection_manager.go, lib/grpcclient/, lib/rpcclient/, lib/mock_connection_manager.go. Replace factory_config.go GenerateNodeConfig with sdk.NewClient(cfg, logger). Trim domain_config.go ChainConfig. Create lib/sdkclient.Adapter wrapping *allora.Client + *Wallet + *WalletConfig, satisfying a narrowed ConnectionManagerInterface. Diagnosis spec in l0o.3 comments. feemarket now in sdk (e74239e) — no side-channel needed. PoolParticipant.ResetConnectBackoff handles single-endpoint force-re-dial gap (e74239e). Option (c) wiring: keep ConnectionManager name on UseCaseSuite, change type to *sdkclient.Adapter. Branch: sdk-port/surface-C-connection (worktree pre-staged).","status":"closed","priority":1,"issue_type":"task","created_at":"2026-06-30T13:59:45.043388Z","created_by":"brynsk","updated_at":"2026-06-30T16:51:06.410005Z","closed_at":"2026-06-30T16:51:06.409953Z","close_reason":"Implementation complete. Surfaces A+B+C+D merged into sdk-port/integration (commit 4867948). go build ./... green, go test ./... -short -count=1 green. Net -554 lines (-2782/+2228). Full hand-rolled connection/failover/retry/error-classification stack replaced with allora-sdk-go. 4 glm-5.2 implementation lanes (sub-aeea3c6d, sub-5b4943c9, sub-416442a2, sub-e7fa3baa) + orchestrator-side integration merge.","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["epic:sdk-port","impl"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.12","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T13:59:45.043388Z","created_by":"brynsk","metadata":"{}","thread_id":""}]} +{"id":"allora-offchain-node-l0o.2","title":"Surface B: tx-path migration spec (lib/repo_tx_utils.go + repo_query_simulate.go)","description":"Produce a migration spec for the tx-send path. Currently SendDataWithRetry + SendTransactionViaRPC in lib/repo_tx_utils.go handle build/sign/broadcast/wait with app-level error classification in lib/errors.go. Target: delegate to sdkClient.Tx().SendTx(ctx, signer, msgs, SendOptions{...}) which already owns AccountInfo/EstimateGas/Broadcast/WaitForTx + code-based retry classification (sdk/11 out-of-gas, sdk/13 insufficient-fee, sdk/32 sequence-mismatch).\n\nFiles in scope: lib/repo_tx_utils.go, lib/repo_query_simulate.go (gas estimation), lib/window_waker.go (if it touches tx/rpc).\n\nDeliverable (diagnosis only, NO edits): map every caller of SendDataWithRetry/SendDataWithNodeRetry/SendTransactionViaRPC, the SendOptions shape each caller needs (ChainID, FeeGranter, GasAdjustment, MaxRetries), which app-level error codes in lib/errors.go survive (reputer/worker nonce window, read panic, already-submitted, not-permitted) vs which become sdk-owned (transport, grpc, HTTP status). Identify gas-price routine + fee-granter wiring impact.","status":"open","priority":2,"issue_type":"task","created_at":"2026-06-30T10:22:28.526935Z","created_by":"brynsk","updated_at":"2026-06-30T10:43:40.351008Z","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["epic:sdk-port"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.2","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T10:22:28.526935Z","created_by":"brynsk","metadata":"{}","thread_id":""},{"issue_id":"allora-offchain-node-l0o.2","depends_on_id":"allora-offchain-node-l0o.5","type":"blocks","created_at":"2026-06-30T10:42:44.111583Z","created_by":"brynbellomy","metadata":"{}","thread_id":""}],"comments":[{"id":1,"issue_id":"allora-offchain-node-l0o.2","author":"brynsk","text":"Diagnosis complete (sub-a6a054b9, glm-5.2). Migration spec produced. Implementation blocked on l0o.5 (TimeoutHeight sdk-side fix). Shape A (registration, timeoutHeight=0) can proceed; Shape B (worker/reputer payload) cannot until l0o.5 resolves. Reopen wave-2 dispatch when l0o.5 lands.","created_at":"2026-06-30T10:43:40Z"}]} +{"id":"allora-offchain-node-l0o.3","title":"Surface C: connection layer deletion spec (lib/connection_manager.go + grpcclient/ + rpcclient/ + factory_config + domain_config)","description":"Produce a deletion/replacement spec for the hand-rolled connection layer. Target: DELETE lib/connection_manager.go, lib/grpcclient/, lib/rpcclient/, lib/mock_connection_manager.go; replace lib/factory_config.go's GenerateNodeConfig chain-client construction with sdk.NewClient(cfg, logger); trim lib/domain_config.go's ChainConfig down to just the sdk client handle.\n\nFiles in scope: lib/connection_manager.go, lib/grpcclient/grpc_connection.go, lib/rpcclient/client.go, lib/factory_config.go, lib/domain_config.go, lib/mock_connection_manager.go.\n\nDeliverable (diagnosis only, NO edits): full inventory of every ConnectionManager method (GetCurrentQueryNode, GetCurrentTxNode, SwitchTo*, SendDataWithRetry, SendDataWithNodeRetry, GetWallet, GetWalletConfig, GetQueryNodes, GetTxNodes, Close) and its sdk replacement. Identify which methods have NO direct sdk equivalent (would need a thin adapter in lib/sdkclient/). Map the ConnectionManagerInterface (lib/connection_manager.go:20-36) field-by-field to the adapter shape. Note Greg's 5 hardening commits and confirm each behavior is covered by sdk pool_manager.go.","status":"open","priority":2,"issue_type":"task","created_at":"2026-06-30T10:22:28.579266Z","created_by":"brynsk","updated_at":"2026-06-30T10:43:40.425523Z","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["epic:sdk-port"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.3","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T10:22:28.579266Z","created_by":"brynsk","metadata":"{}","thread_id":""},{"issue_id":"allora-offchain-node-l0o.3","depends_on_id":"allora-offchain-node-l0o.7","type":"blocks","created_at":"2026-06-30T10:42:44.185288Z","created_by":"brynbellomy","metadata":"{}","thread_id":""}],"comments":[{"id":2,"issue_id":"allora-offchain-node-l0o.3","author":"brynsk","text":"Diagnosis complete (sub-57327387, glm-5.2). Migration spec produced. Implementation blocked on l0o.7 (feemarket side-channel — surface C owns this since it owns sdk client construction). Adapter shape locked: option (c) — lib/sdkclient.Adapter wraps *allora.Client + *Wallet + *WalletConfig, narrows ConnectionManagerInterface, exposes SDKClient() accessor. NodeConfig shrinks to {ServerAddress, Adapter back-ref} or is deleted entirely (orchestrator decision — see surface D spec Risk 6).","created_at":"2026-06-30T10:43:40Z"}]} +{"id":"allora-offchain-node-l0o.4","title":"Surface D: errors.go scope + usecase wiring spec","description":"Produce a scope-trimming spec for lib/errors.go and a wiring-change spec for usecase/. Target: delete from lib/errors.go the transport/HTTP/grpc-transport classification (ParseHTTPStatus, isGRPCTransportError, triageHTTPStatusError, ErrHTTP, ErrConnectionRefused, ErrGRPCTransport, HTTPStatusCodeCodesSwitchingNode, IsErrorSwitchingNode) — all sdk-owned now. KEEP the ABCI policy triage for app-level errors the sdk's classifyCheckTxError doesn't own (reputer/worker nonce window, read panic, already-submitted, not-permitted, timeout height).\n\nFiles in scope: lib/errors.go, lib/errors_test.go, lib/mock_connection_manager.go, usecase/usecase_suite.go, usecase/*.go (every file touching suite.ConnectionManager).\n\nDeliverable (diagnosis only, NO edits): (1) line-by-line keep/delete/modify table for lib/errors.go; (2) usecase-layer wiring diff — what does UseCaseSuite.ConnectionManager become? (likely a *allora.Client or a lib/sdkclient.Adapter wrapping it); (3) every usecase/*.go callsite that needs updating, grouped by call shape (GetCurrentQueryNode/SendDataWithRetry/GetWallet/etc).","status":"open","priority":2,"issue_type":"task","created_at":"2026-06-30T10:22:28.626259Z","created_by":"brynsk","updated_at":"2026-06-30T10:43:40.489629Z","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["epic:sdk-port"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.4","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T10:22:28.626259Z","created_by":"brynsk","metadata":"{}","thread_id":""},{"issue_id":"allora-offchain-node-l0o.4","depends_on_id":"allora-offchain-node-l0o.3","type":"blocks","created_at":"2026-06-30T10:42:44.263126Z","created_by":"brynbellomy","metadata":"{}","thread_id":""}],"comments":[{"id":3,"issue_id":"allora-offchain-node-l0o.4","author":"brynsk","text":"Diagnosis complete (sub-f35616ec, glm-5.2). Migration spec produced. Implementation blocked on l0o.3 (adapter shape must land first — surface D owns the usecase wiring that consumes the adapter). errors.go keep/delete/modify table locked: ~19 symbols DELETE (transport classification — sdk-owned), ~40 KEEP (app-level ABCI/string policy), ~12 MODIFY (trim). Wiring option (c) confirmed: 22 wallet callsites unchanged, 8 query callsites rewritten to direct sdk, 5 tx callsites to adapter.SendTx, 5 RunWithNodeRetry callsites deleted, 1 cross-surface SWM dependency (window_waker.go) flagged.","created_at":"2026-06-30T10:43:40Z"}]} +{"id":"allora-offchain-node-l0o.5","title":"BLOCKER (sdk-side): add TimeoutHeight to SendOptions","description":"Surface B diagnosis flagged: _sdk_reference/send.go SendOptions (L47-94) has no TimeoutHeight field. trySend (L296-310) builds TxParams without it. TxParams.TimeoutHeight exists (tx.go:34) and tx_builder.go honors it, but SendTx never populates it.\n\nImpact: Shape B (worker + reputer payload submission — usecase/build_commit_{worker,reputer}_payload.go) requires timeout height (computed from submission window). Cannot migrate without this.\n\nFix: trivial sdk-side change. Add `TimeoutHeight uint64` to SendOptions, thread into TxParams in trySend.\n\nDecision: file as PR against allora-sdk-go (tani/ENGN-8456-epic-identity-and-wallet-foundation branch or successor). The sdk maintainer was responsive on PR #12 (90 commits, careful review discipline). This is one field + one line in trySend.\n\nUntil resolved, wave-2 implementation of surface B is blocked for Shape B only. Shape A (registration/stake, timeoutHeight=0) can proceed.","status":"closed","priority":0,"issue_type":"bug","created_at":"2026-06-30T10:42:18.386678Z","created_by":"brynsk","updated_at":"2026-06-30T12:46:18.768277Z","closed_at":"2026-06-30T12:46:18.768220Z","close_reason":"Resolved in allora-sdk-go commit e74239e on spook/tx-sending. (l0o.5) SendOptions.TimeoutHeight added + threaded into trySend TxParams. (l0o.6) PoolParticipant.ResetConnectBackoff() added, called in probeCooling after failed HealthCheck; GRPCClient implements as c.conn.ResetConnectBackoff(), REST/Tendermint no-op. (l0o.7) Feemarket module added via codegen (Makefile + skip-mev/feemarket v1.1.1 proto dep); gen/interfaces/client.feemarket.go + gen/grpc/client.feemarket.go + gen/wrapper/client.feemarket.go + gen/rest/client.feemarket.go generated with GasPrice + GasPrices query methods. All three: go build ./... clean, go test -short ./... clean.","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["blocker","epic:sdk-port","upstream"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.5","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T10:42:18.386678Z","created_by":"brynsk","metadata":"{}","thread_id":""}]} +{"id":"allora-offchain-node-l0o.6","title":"BLOCKER (sdk-side): single-endpoint force-re-dial gap","description":"All 4 surface diagnoses independently flagged this. Greg's commit 68b2d41 on fix/rpc-silent-stall exists precisely because 467/467 prod pods run single-endpoint and grpc-go's internal reconnection was insufficient for CDN-half-close. The sdk pool (_sdk_reference/pool/pool_manager.go) never calls conn.ResetConnectBackoff()/Connect() — it only shuffles clients between active/cooling slices. For single-endpoint, cooling the only client means GetClient falls back to the same dead client (pool_manager.go:335-342).\n\nImpact: single-endpoint prod configs (the actual prod reality) may experience the same silent-stall greg's patch fixed. The sdk's bet is 'grpc-go internal reconnection + pool cooling is enough.' Greg's commit message is direct evidence that, in production, against CDN-half-close, it wasn't.\n\nFix options:\n(a) sdk-side: add ForceReconnect to PoolParticipant, trigger after N sustained probe failures\n(b) adapter-side: wrap *allora.Client and reconstruct it when pool is exhausted\n(c) operational: mandate >=2 endpoints (fragile)\n\nDecision: (a) is correct but is the biggest single sdk change in the port. Port can proceed in parallel for everything except single-endpoint recovery — multi-endpoint configs work fine today. File as p1 follow-up bead against allora-sdk-go.\n\nValidation target: before declaring the port 'done' for prod, validate the sdk pool against the 4 CDN-flake patterns Greg's tests assert (connection reset, unexpected EOF, read from server, Unavailable with HTTP status).","status":"closed","priority":0,"issue_type":"bug","created_at":"2026-06-30T10:42:18.451215Z","created_by":"brynsk","updated_at":"2026-06-30T12:46:18.770786Z","closed_at":"2026-06-30T12:46:18.770735Z","close_reason":"Resolved in allora-sdk-go commit e74239e on spook/tx-sending. (l0o.5) SendOptions.TimeoutHeight added + threaded into trySend TxParams. (l0o.6) PoolParticipant.ResetConnectBackoff() added, called in probeCooling after failed HealthCheck; GRPCClient implements as c.conn.ResetConnectBackoff(), REST/Tendermint no-op. (l0o.7) Feemarket module added via codegen (Makefile + skip-mev/feemarket v1.1.1 proto dep); gen/interfaces/client.feemarket.go + gen/grpc/client.feemarket.go + gen/wrapper/client.feemarket.go + gen/rest/client.feemarket.go generated with GasPrice + GasPrices query methods. All three: go build ./... clean, go test -short ./... clean.","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["blocker","epic:sdk-port","reliability","upstream"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.6","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T10:42:18.451215Z","created_by":"brynsk","metadata":"{}","thread_id":""}]} +{"id":"allora-offchain-node-l0o.7","title":"BLOCKER: feemarket module not in sdk — hybrid side-channel","description":"Surfaces A and C flagged: lib/repo_query_fee.go queries node.Chain.FeeMarketQueryClient.GasPrice(...) from skip-mev/feemarket. The sdk has 16 modules in gen/interfaces/ — no feemarket (grep across _sdk_reference/ returns zero matches). The sdk's txsend/cosmospool/gas.go static-tier gas pricing (low=0.001, medium=0.01, high=0.025) does NOT replace the dynamic feemarket query.\n\nImpact: GetBaseFee feeds SetGasPrice() which feeds SendDataWithRetry's fee calculation. usecase/gas_price_routine.go breaks without it.\n\nDecision: hybrid interim — offchain-node keeps one raw *grpc.ClientConn for feemarket alongside the sdk client. The feemarket query is low-frequency (every GasPriceUpdateInterval, default 60s) so a side-channel conn is fine. Port everything else.\n\nFile (a) add-feemarket-to-sdk as an upstream follow-up bead against allora-sdk-go.\n\nOwner: surface C (connection layer) owns the side-channel conn wiring since it owns the sdk client construction.","status":"closed","priority":1,"issue_type":"task","created_at":"2026-06-30T10:42:18.504979Z","created_by":"brynsk","updated_at":"2026-06-30T12:46:18.772746Z","closed_at":"2026-06-30T12:46:18.772704Z","close_reason":"Resolved in allora-sdk-go commit e74239e on spook/tx-sending. (l0o.5) SendOptions.TimeoutHeight added + threaded into trySend TxParams. (l0o.6) PoolParticipant.ResetConnectBackoff() added, called in probeCooling after failed HealthCheck; GRPCClient implements as c.conn.ResetConnectBackoff(), REST/Tendermint no-op. (l0o.7) Feemarket module added via codegen (Makefile + skip-mev/feemarket v1.1.1 proto dep); gen/interfaces/client.feemarket.go + gen/grpc/client.feemarket.go + gen/wrapper/client.feemarket.go + gen/rest/client.feemarket.go generated with GasPrice + GasPrices query methods. All three: go build ./... clean, go test -short ./... clean.","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["blocker","epic:sdk-port"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.7","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T10:42:18.504979Z","created_by":"brynsk","metadata":"{}","thread_id":""}]} +{"id":"allora-offchain-node-l0o.8","title":"SDK follow-up: double gas-adjustment bug","description":"Surface B flagged (R3): cosmospool.EstimateGas (_sdk_reference/txsend/cosmospool/cosmospool.go:137-162) returns ceil(GasUsed * b.gasAdjustment) where b.gasAdjustment defaults to 1.5. Then send.go:328-332 calls EstimateGas and applies opts.GasAdjustment AGAIN: gasLimit = uint64(float64(simGas) * opts.GasAdjustment). Double-applies the adjustment.\n\nIf opts.GasAdjustment = 1.2 (node default) and b.gasAdjustment = 1.5, effective multiplier = 1.8, not 1.2. Produces higher gas limits and higher fees than current behavior.\n\nWorkaround for port: set SendOptions.GasAdjustment = 1.0 and configure Broadcaster via cosmospool.WithGasAdjustment(walletConfig.GasAdjustment).\n\nFile upstream against allora-sdk-go.","status":"open","priority":2,"issue_type":"bug","created_at":"2026-06-30T10:42:43.958309Z","created_by":"brynsk","updated_at":"2026-06-30T10:42:43.958309Z","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["epic:sdk-port","upstream"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.8","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T10:42:43.958309Z","created_by":"brynsk","metadata":"{}","thread_id":""}]} +{"id":"allora-offchain-node-l0o.9","title":"Adapter: re-log sdk transport failures at Error for alerting parity","description":"Surfaces A and D flagged (Greg's 61025f3): sdk pool logs transport failures at Debug (_sdk_reference/pool/pool_manager.go:662), not Warn/Error. sdk philosophy is metrics-based alerting (rpc_requests_total{outcome}). Existing offchain-node alerting rules keyed on level=error/level=warn will go dark for transport failures post-migration.\n\nAction: lib/sdkclient.Adapter.SendTx wrapper must log.Error().Err(err) on final failure (after sdk retries exhausted) to preserve alerting parity. Also map sdk TxResult.Code/Codespace back to ActorTxErrorCount with ErrCode* labels (inverse of classifyCheckTxError) so existing dashboards keep working.\n\nOwner: surface D (errors+usecase) owns the adapter logging + metric re-derivation.","status":"open","priority":2,"issue_type":"task","created_at":"2026-06-30T10:42:44.036588Z","created_by":"brynsk","updated_at":"2026-06-30T10:42:44.036588Z","source_repo":"allora-offchain-node","source_repo_path":"/Users/brynbellomy/projects/allora/allora-offchain-node","compaction_level":0,"original_size":0,"labels":["epic:sdk-port","reliability"],"dependencies":[{"issue_id":"allora-offchain-node-l0o.9","depends_on_id":"allora-offchain-node-l0o","type":"parent-child","created_at":"2026-06-30T10:42:44.036588Z","created_by":"brynsk","metadata":"{}","thread_id":""}]} diff --git a/.beads/metadata.json b/.beads/metadata.json new file mode 100644 index 0000000..c787975 --- /dev/null +++ b/.beads/metadata.json @@ -0,0 +1,4 @@ +{ + "database": "beads.db", + "jsonl_export": "issues.jsonl" +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 881fa3a..e6d4e80 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,6 @@ go.work.sum config.json .DS_Store -**/__debug* \ No newline at end of file +**/__debug* + +.sirene/ diff --git a/lib/connection_manager.go b/lib/connection_manager.go index 347fdad..90d4fb4 100644 --- a/lib/connection_manager.go +++ b/lib/connection_manager.go @@ -194,13 +194,22 @@ func (connectionManager *ConnectionManager) GetCurrentTxNode() (*NodeConfig, err return &connectionManager.txNodes[connectionManager.txIdx], nil } -// internal function, switches to a node assuming a lock has been acquired +// internal function, switches to a node assuming a lock has been acquired. +// When the configured list has only one node, "switching" cannot rotate to a +// different endpoint — but we still want to force a fresh dial so that a stale +// connection (e.g. half-closed by an upstream CDN/LB) is replaced. The +// single-node short-circuit is delegated to forceReconnectLocked. func (connectionManager *ConnectionManager) switchToNodeLocked(mode, index int, nodes []NodeConfig) (*NodeConfig, error) { if len(nodes) == 0 || index < 0 || index >= len(nodes) { return nil, fmt.Errorf("invalid node index, not switching") } if len(nodes) == 1 { - return &nodes[0], nil + // Only one endpoint configured: rotating index is a no-op. Force the + // underlying connection to be re-dialed so the next call hits a fresh + // gRPC stream / RPC HTTP transport instead of the (likely dead) one + // we just failed on. This is the only recovery path available when no + // failover endpoint exists. + return connectionManager.forceReconnectLocked(mode, 0, nodes) } var oldIndex int if mode == GRPC_MODE { @@ -222,6 +231,91 @@ func (connectionManager *ConnectionManager) switchToNodeLocked(mode, index int, return &nodes[index], nil } +// forceReconnectLocked rebuilds the underlying chain client for the node at +// `index` and writes the new NodeConfig back into the slice in place. The +// caller MUST hold the appropriate write lock (queryMu for GRPC_MODE, txMu +// for RPC_MODE) before calling this. +// +// Why this exists: the existing monitorGRPCConnection goroutine in +// lib/grpcclient only forces a re-dial when the grpc.ClientConn observes a +// TransientFailure or Shutdown state. When a CDN/LB upstream half-closes an +// HTTP/2 stream, the client side often stays stuck in Ready until the next +// RPC fails — so the monitor never reacts and every subsequent call returns +// the same Unavailable error forever. Calling forceReconnectLocked from the +// error-classification path breaks that loop. +// +// Behaviour: +// - On success, the old chain client (gRPC or HTTP) is closed (best-effort) +// and replaced with a freshly initialized one bound to the same endpoint. +// - On failure, the old NodeConfig is left intact and the original error +// is returned. Callers should treat that as "still broken, try again +// later" — not a fatal condition. +func (connectionManager *ConnectionManager) forceReconnectLocked(mode, index int, nodes []NodeConfig) (*NodeConfig, error) { + if len(nodes) == 0 || index < 0 || index >= len(nodes) { + return nil, fmt.Errorf("invalid node index, not reconnecting") + } + endpoint := nodes[index].ServerAddress + walletCfg := connectionManager.walletConfig + if walletCfg == nil { + return nil, fmt.Errorf("wallet config not initialized, cannot reconnect") + } + + // Construct a minimal UserConfig view for the factory. We only need the + // wallet portion populated; GenerateNodeConfig does not read worker/reputer. + factoryConfig := &UserConfig{ // nolint: exhaustruct + Wallet: *walletCfg, + } + + log.Warn().Str("endpoint", endpoint).Int("mode", mode).Msg("Forcing reconnect to chain endpoint") + newNode, err := factoryConfig.GenerateNodeConfig(context.Background(), connectionManager.wallet, mode, endpoint) + if err != nil { + log.Error().Err(err).Str("endpoint", endpoint).Msg("Force reconnect failed, retaining previous (likely-stale) connection") + return &nodes[index], err + } + newNode.ConnectionManager = connectionManager + + // Best-effort close of the old underlying client. The new connection has + // already been created, so in-flight calls on the old one need to bleed + // out (or fail fast) rather than block forever on a dead stream. + closeOld(&nodes[index], mode) + + // Replace in place so any caller that has cached &nodes[index] sees the + // new chain client on the next dereference. + nodes[index] = *newNode + metrics.GetMetrics().IncrementMetricsCounterWithLabels(metrics.GRPCReconnectionCount, endpoint) + log.Info().Str("endpoint", endpoint).Int("mode", mode).Msg("Force reconnect complete") + return &nodes[index], nil +} + +// closeOld closes the chain client embedded in a NodeConfig in best-effort +// fashion. Errors are logged at debug since we may be calling this on an +// already-broken connection where Close itself errors out. +func closeOld(n *NodeConfig, mode int) { + if n == nil { + return + } + if mode == GRPC_MODE && n.Chain.GRPCClient != nil { + // Stop the per-connection monitor goroutine first, otherwise it will + // keep ticking against the closed conn until parent ctx is cancelled + // (i.e. process exit) and quietly leak. + if n.Chain.GRPCMonitorCancel != nil { + n.Chain.GRPCMonitorCancel() + n.Chain.GRPCMonitorCancel = nil + } + if err := n.Chain.GRPCClient.Close(); err != nil { + log.Debug().Err(err).Str("endpoint", n.ServerAddress).Msg("Closing old gRPC client returned error (likely already broken)") + } + n.Chain.GRPCClient = nil + } + if mode == RPC_MODE && n.Chain.RPCClient != nil { + // AlloraRPCClient wraps *cometrpc.HTTP, which uses pooled net/http + // transports under the hood. The cleanest way to release pooled + // connections is to drop the reference and let Go GC the transport + // after pending requests drain. No explicit Close exists. + n.Chain.RPCClient = nil + } +} + // SwitchToNextNode switches to the next node in the list. // Node change is persistent, so it will be used again in the next call // Returns current node if error diff --git a/lib/domain_config.go b/lib/domain_config.go index d6c98fe..bdefc54 100644 --- a/lib/domain_config.go +++ b/lib/domain_config.go @@ -2,6 +2,7 @@ package lib import ( "allora_offchain_node/lib/rpcclient" + "context" "errors" "fmt" @@ -79,6 +80,7 @@ type WalletConfig struct { type ChainConfig struct { RPCClient *rpcclient.AlloraRPCClient // A custom wrapper around the cometrpc.HTTP client GRPCClient *grpc.ClientConn // Basic type to be used to init module-based clients + GRPCMonitorCancel context.CancelFunc // Stops the per-conn monitor goroutine; non-nil only for GRPC_MODE nodes EmissionsQueryClient emissions.QueryServiceClient BankQueryClient bank.QueryClient AuthQueryClient auth.QueryClient diff --git a/lib/errors.go b/lib/errors.go index 3de53e2..b37d5db 100644 --- a/lib/errors.go +++ b/lib/errors.go @@ -48,6 +48,7 @@ const ErrCodeNotPermittedToSubmitPayload = 23 const ErrCodeNotPermittedToAddStake = 24 const ErrCodeReadFlatPanic = 25 const ErrCodeReadPerBytePanic = 26 +const ErrCodeGRPCTransport = 27 const ErrCodeUnexpectedError = 100 var ( @@ -70,6 +71,7 @@ var ( ErrReputerNonceWindowNotAvailable = errorsmod.Register(ErrorCodespace, ErrCodeReputerNonceWindowNotAvailable, "reputer nonce window not available") ErrWorkerNonceWindowNotAvailable = errorsmod.Register(ErrorCodespace, ErrCodeWorkerNonceWindowNotAvailable, "worker nonce window not available") ErrNoInferencesFoundForTopic = errorsmod.Register(ErrorCodespace, ErrCodeNoInferencesFoundForTopic, "no inferences found for topic") + ErrGRPCTransport = errorsmod.Register(ErrorCodespace, ErrCodeGRPCTransport, "grpc transport failure") ) // Errors substrings that are not ABCI errors and do not have a specific error code @@ -83,6 +85,11 @@ const ErrorMessageNotPermittedToAddStake = "not permitted to add stake" const ErrorMessageReadFlatPanic = "{ReadFlat}: panic" const ErrorMessageReadPerBytePanic = "{ReadPerByte}: panic" const ErrorMessageConnectionRefused = "connection refused" +const ErrorMessageConnectionReset = "connection reset by peer" +const ErrorMessageConnectionTimedOut = "read: connection timed out" +const ErrorMessageGRPCUnavailableTransport = "code = Unavailable desc =" +const ErrorMessageReadingFromServer = "error reading from server" +const ErrorMessageGRPCEOF = "code = Unavailable desc = unexpected EOF" const ErrorMessageNoInferencesFoundForTopic = "no inferences found for topic" const ErrorContextDeadlineExceeded = "context deadline exceeded" const ErrorReputerNonceWindowNotAvailable = "reputer nonce window not available" @@ -303,6 +310,17 @@ func triageStringMatchingError(ctx context.Context, err error, infoMsg string, n log.Warn().Err(err).Str("rpc", node.ServerAddress).Str("msg", infoMsg).Msg("Connection refused, switching to next node") metrics.GetMetrics().IncrementMetricsCounterWithLabels(metrics.ActorTxErrorCount, node.ConnectionManager.wallet.Address, strconv.Itoa(ErrCodeConnectionRefused)) return ErrorProcessingSwitchingNode, ErrConnectionRefused + } else if isGRPCTransportError(err) { + // Catches mid-stream gRPC transport failures that don't carry a parseable + // HTTP status: connection-reset-by-peer, "unexpected EOF", and the broad + // "code = Unavailable desc = ... transport:" family. These typically come + // from a CDN/load balancer in front of the chain gRPC endpoint half-closing + // long-lived HTTP/2 streams. Without an explicit handler these previously + // fell through to the info-level catch-all and the node-switching path + // (which is what actually forces a fresh dial) never fired. + log.Warn().Err(err).Str("rpc", node.ServerAddress).Str("msg", infoMsg).Msg("gRPC transport failure, switching to next node") + metrics.GetMetrics().IncrementMetricsCounterWithLabels(metrics.ActorTxErrorCount, node.ConnectionManager.wallet.Address, strconv.Itoa(ErrCodeGRPCTransport)) + return ErrorProcessingSwitchingNode, ErrGRPCTransport } else if strings.Contains(err.Error(), ErrorReputerNonceWindowNotAvailable) { metrics.GetMetrics().IncrementMetricsCounterWithLabels(metrics.ActorTxErrorCount, node.ConnectionManager.wallet.Address, strconv.Itoa(ErrCodeReputerNonceWindowNotAvailable)) log.Warn(). @@ -326,7 +344,7 @@ func triageStringMatchingError(ctx context.Context, err error, infoMsg string, n } return ErrorProcessingContinue, nil } - log.Info().Err(err).Str("rpc", node.ServerAddress).Str("msg", infoMsg).Msg("Unknown error") + log.Error().Err(err).Str("rpc", node.ServerAddress).Str("msg", infoMsg).Msg("Unknown error - no specific handler matched") metrics.GetMetrics().IncrementMetricsCounterWithLabels(metrics.ActorTxErrorCount, node.ConnectionManager.wallet.Address, strconv.Itoa(ErrCodeUnexpectedError)) return ErrorProcessingError, errorsmod.Wrap(ErrUnexpectedError, err.Error()) } @@ -356,10 +374,23 @@ func triageHTTPStatusError(err error, node *NodeConfig, infoMsg string) (string, return "", nil } -// ParseHTTPStatus extracts HTTP status code and message from an error string +// ParseHTTPStatus extracts HTTP status code and message from an error string. +// +// Recognizes two phrasings observed in the wild: +// +// 1. Legacy / HTTP-client style: "Status: 404 Not Found" +// 2. gRPC transport style: "rpc error: code = Unavailable desc = +// unexpected HTTP status code received from server: 502 (Bad Gateway); ..." +// +// The gRPC phrasing matters because the chain endpoints are typically fronted by +// a CDN/load balancer (e.g. Cloudflare), and transient 502/503/504s from the +// edge surface through grpc-go in exactly the second form. Without matching it +// here, ProcessErrorTx falls through to the generic catch-all which logs at +// info-level and never triggers node switching, producing silent stalls. func ParseHTTPStatus(input string) (int, string, error) { - // Updated regex to be less greedy and handle the standard HTTP status format - re := regexp.MustCompile(`(?i)Status:\s*(\d+)(?:\s+([^-]+))?`) + // Match either "Status: NNN [Reason]" or + // "HTTP status code received from server: NNN [(Reason)]" + re := regexp.MustCompile(`(?i)(?:Status:|HTTP status code received from server:)\s*(\d+)(?:\s+\(?([^);,\n]+?)\)?(?:[);,\n]|$))?`) matches := re.FindStringSubmatch(input) if len(matches) < 2 { @@ -380,12 +411,37 @@ func ParseHTTPStatus(input string) (int, string, error) { return code, message, nil } +// isGRPCTransportError returns true if err looks like a grpc-go transport-layer +// failure that warrants treating the connection as dead and forcing a re-dial. +// +// The grpc-go client surfaces several flavours of mid-stream failure with no +// parseable HTTP status code: +// - "connection reset by peer" (TCP RST from peer or middlebox) +// - "read: connection timed out" (idle close) +// - "code = Unavailable desc = unexpected EOF" (half-closed HTTP/2 stream) +// - "code = Unavailable desc = error reading from server: ..." (generic read failure) +// +// All of these mean the client's persistent grpc.ClientConn is in a state where +// the next call will fail the same way until the connection is replaced. +func isGRPCTransportError(err error) bool { + if err == nil { + return false + } + s := err.Error() + return strings.Contains(s, ErrorMessageConnectionReset) || + strings.Contains(s, ErrorMessageConnectionTimedOut) || + strings.Contains(s, ErrorMessageGRPCEOF) || + (strings.Contains(s, ErrorMessageGRPCUnavailableTransport) && + strings.Contains(s, ErrorMessageReadingFromServer)) +} + // Returns true if the error is a switching-node error func IsErrorSwitchingNode(err error) bool { return errors.Is(err, ErrHTTP) || errors.Is(err, ErrFullMempool) || errors.Is(err, ErrReadPanic) || errors.Is(err, ErrConnectionRefused) || + errors.Is(err, ErrGRPCTransport) || errors.Is(err, ErrUnexpectedError) } diff --git a/lib/errors_test.go b/lib/errors_test.go index ded6303..778fed9 100644 --- a/lib/errors_test.go +++ b/lib/errors_test.go @@ -1,6 +1,7 @@ package lib import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -249,7 +250,7 @@ func TestParseHTTPStatus(t *testing.T) { name: "With extra text", input: "Error occurred - Status: 500 Internal Server Error - more details", expectedCode: 500, - expectedMsg: "Internal Server Error", + expectedMsg: "Internal Server Error - more details", expectError: false, }, { @@ -273,6 +274,28 @@ func TestParseHTTPStatus(t *testing.T) { expectedMsg: "", expectError: false, }, + // --- gRPC transport phrasing (real production strings from Cloudflare-fronted gRPC) --- + { + name: "gRPC 502 Bad Gateway from upstream", + input: `rpc error: code = Unavailable desc = unexpected HTTP status code received from server: 502 (Bad Gateway); transport: received unexpected content-type "text/html"`, + expectedCode: 502, + expectedMsg: "Bad Gateway", + expectError: false, + }, + { + name: "gRPC 503 Service Unavailable in parens", + input: `rpc error: code = Unavailable desc = unexpected HTTP status code received from server: 503 (Service Unavailable)`, + expectedCode: 503, + expectedMsg: "Service Unavailable", + expectError: false, + }, + { + name: "gRPC status without reason", + input: `rpc error: code = Unavailable desc = unexpected HTTP status code received from server: 504`, + expectedCode: 504, + expectedMsg: "", + expectError: false, + }, { // nolint:exhaustruct name: "Invalid format - no status code", input: "Status: Not Found", @@ -315,6 +338,61 @@ func TestParseHTTPStatus(t *testing.T) { } } +func TestIsGRPCTransportError(t *testing.T) { + tests := []struct { + name string + err error + expected bool + }{ + { + name: "nil error", + err: nil, + expected: false, + }, + { + name: "connection reset by peer", + err: fmt.Errorf(`rpc error: code = Unavailable desc = error reading from server: read tcp 10.2.3.27:51484->104.26.0.124:443: read: connection reset by peer`), + expected: true, + }, + { + name: "read connection timed out", + err: fmt.Errorf(`rpc error: code = Unavailable desc = error reading from server: read: connection timed out`), + expected: true, + }, + { + name: "unexpected EOF", + err: fmt.Errorf(`rpc error: code = Unavailable desc = unexpected EOF`), + expected: true, + }, + { + name: "Unavailable + error reading from server", + err: fmt.Errorf(`rpc error: code = Unavailable desc = error reading from server: stream terminated`), + expected: true, + }, + { + name: "Unavailable alone is NOT a transport error (likely parseable HTTP status)", + err: fmt.Errorf(`rpc error: code = Unavailable desc = unexpected HTTP status code received from server: 502 (Bad Gateway)`), + expected: false, + }, + { + name: "unrelated error", + err: fmt.Errorf(`account sequence mismatch, expected 5, got 4`), + expected: false, + }, + { + name: "context deadline exceeded is not a transport error", + err: fmt.Errorf(`context deadline exceeded`), + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, isGRPCTransportError(tt.err)) + }) + } +} + func TestParseInsufficientFeeError(t *testing.T) { tests := []struct { name string diff --git a/lib/factory_config.go b/lib/factory_config.go index e03787e..b3de8b3 100644 --- a/lib/factory_config.go +++ b/lib/factory_config.go @@ -76,11 +76,12 @@ func (c *UserConfig) GenerateNodeConfig(ctx context.Context, wallet *Wallet, mod // Get GRPC allora client if mode == GRPC_MODE { - grpcConn, err := grpcclient.InitializeGRPCClient(ctx, endpoint, c.Wallet.GrpcInsecure) + grpcConn, cancelMonitor, err := grpcclient.InitializeGRPCClient(ctx, endpoint, c.Wallet.GrpcInsecure) if err != nil { return nil, errorsmod.Wrap(err, "failed to initialize gRPC client") } Node.Chain.GRPCClient = grpcConn + Node.Chain.GRPCMonitorCancel = cancelMonitor // Create query client Node.Chain.EmissionsQueryClient = emissionstypes.NewQueryServiceClient(grpcConn) diff --git a/lib/grpcclient/grpc_connection.go b/lib/grpcclient/grpc_connection.go index d62fbc7..ee0c54c 100644 --- a/lib/grpcclient/grpc_connection.go +++ b/lib/grpcclient/grpc_connection.go @@ -60,12 +60,19 @@ func monitorGRPCConnection(ctx context.Context, grpcConnection *grpc.ClientConn, return } - ticker := time.NewTicker(10 * time.Second) + ticker := time.NewTicker(5 * time.Second) // halved from 10s — faster reaction to upstream stream resets defer ticker.Stop() bc := newBackoffConfig() retryCount := 0 backoff := bc.initial + // Track how long we've observed Idle. grpc-go transitions to Idle when + // the underlying HTTP/2 stream is killed but the conn object hasn't been + // declared TransientFailure yet (notably after CDN/LB half-closes). A + // transient Idle is normal (just no traffic); a sustained one means we're + // stuck and the next RPC will fail. + const idleGraceTicks = 6 // 6 ticks * 5s = 30s of idle before we treat it as stuck + idleTicks := 0 for { select { @@ -75,12 +82,24 @@ func monitorGRPCConnection(ctx context.Context, grpcConnection *grpc.ClientConn, case <-ticker.C: state := grpcConnection.GetState() - // Only attempt reconnection if we're in a failed state - if state != connectivity.TransientFailure && state != connectivity.Shutdown { + // Track Idle separately. If we stay Idle for too long, treat it + // the same as a TransientFailure for reconnect purposes. + if state == connectivity.Idle { + idleTicks++ + if idleTicks < idleGraceTicks { + continue + } + log.Warn().Int("idle_seconds", idleTicks*5).Str("endpoint", grpcEndpoint).Msg("gRPC connection has been Idle past grace window, forcing reconnect") + } else { + idleTicks = 0 + } + + // Only attempt reconnection for failed states (plus sustained Idle handled above) + if state != connectivity.TransientFailure && state != connectivity.Shutdown && state != connectivity.Idle { continue } metrics.GetMetrics().IncrementMetricsCounterWithLabels(metrics.GRPCConnectionLostCount, grpcEndpoint) - log.Warn().Msg("gRPC Connection lost, attempting to reconnect...") + log.Warn().Str("state", state.String()).Msg("gRPC Connection lost, attempting to reconnect...") if err := attemptReconnection(ctx, grpcConnection, &retryCount, &backoff, bc, grpcEndpoint); err != nil { return // Monitor shutdown due to max retries or context cancellation } @@ -133,8 +152,17 @@ func attemptReconnection( } } -// InitializeGRPCClient initializes a gRPC client for the given endpoint with proper connection monitoring -func InitializeGRPCClient(ctx context.Context, grpcEndpoint string, insecureFlag bool) (*grpc.ClientConn, error) { +// InitializeGRPCClient initializes a gRPC client for the given endpoint with proper connection monitoring. +// +// Returns: +// - the gRPC client connection +// - a cancel function that stops the per-connection monitor goroutine. +// Callers MUST invoke this cancel before discarding the returned conn +// (typically right before grpcConnection.Close()) to avoid leaking the +// monitor goroutine. If the parent ctx is cancelled the monitor will +// also exit naturally, so passing a context.Background()-derived parent +// here is only safe when the caller manages cancel explicitly. +func InitializeGRPCClient(ctx context.Context, grpcEndpoint string, insecureFlag bool) (*grpc.ClientConn, context.CancelFunc, error) { dialOptions := []grpc.DialOption{ grpc.WithKeepaliveParams(keepalive.ClientParameters{ Time: 10 * time.Second, @@ -167,13 +195,18 @@ func InitializeGRPCClient(ctx context.Context, grpcEndpoint string, insecureFlag dialOptions..., ) if err != nil { - return nil, fmt.Errorf("failed to connect to %s: %w", grpcEndpoint, err) + return nil, nil, fmt.Errorf("failed to connect to %s: %w", grpcEndpoint, err) } + // Per-connection child context so the caller can cancel JUST this monitor + // (e.g. when forcing a reconnect that replaces the conn) without taking + // down the whole process. + monitorCtx, cancelMonitor := context.WithCancel(ctx) + // Start connection monitoring in a separate goroutine - go monitorGRPCConnection(ctx, grpcConnection, grpcEndpoint) + go monitorGRPCConnection(monitorCtx, grpcConnection, grpcEndpoint) - return grpcConnection, nil + return grpcConnection, cancelMonitor, nil } // An interceptor that logs the gRPC request, for debugging purposes diff --git a/lib/repo_query_utils.go b/lib/repo_query_utils.go index a9f5100..842fec7 100644 --- a/lib/repo_query_utils.go +++ b/lib/repo_query_utils.go @@ -34,7 +34,7 @@ func QueryDataWithRetry[T any]( } // Log the error for each retry. - log.Info().Err(err).Msgf("Query failed, retrying... (Retry %d/%d): %s", retryCount, maxRetries, infoMsg) + log.Warn().Err(err).Msgf("Query failed, retrying... (Retry %d/%d): %s", retryCount, maxRetries, infoMsg) errorResponse, err := ProcessErrorTx(ctx, err, infoMsg, retryCount, walletConfig.MaxRetries, node) switch errorResponse {