diff --git a/.changeset/dashboard-gasless-wallet-fallback.md b/.changeset/dashboard-gasless-wallet-fallback.md new file mode 100644 index 0000000000..4d551c6722 --- /dev/null +++ b/.changeset/dashboard-gasless-wallet-fallback.md @@ -0,0 +1,5 @@ +--- +"@anticapture/dashboard": minor +--- + +Offer gasless votes only when the relayer's GET /relay/status answers available: true, and keep every other case on the voter's own wallet: the ballot says when gas-free voting is unavailable or a relayed attempt was refused and switches its button to "Vote with wallet", lets a voter opt out of the relayer at any time, and turns a relay the relayer never confirmed into a "Vote with my wallet instead" button once the chain shows no vote from that address. Delegation gets the same guarantee: a refused or unconfirmed gas-free delegation moves that account to its wallet with a "Delegate with my wallet" button. The fallback state follows the voter across account switches and remounts, a relayed vote whose hash never confirms can also fall back once the chain shows no vote and no pending relay, a wallet vote that finds the relayed vote already counted reports it as cast, and the relayer status refreshes every minute while a page is open diff --git a/.changeset/gateful-relayer-status.md b/.changeset/gateful-relayer-status.md new file mode 100644 index 0000000000..ba995c2ff3 --- /dev/null +++ b/.changeset/gateful-relayer-status.md @@ -0,0 +1,5 @@ +--- +"@anticapture/gateful": minor +--- + +Always answer GET /{dao}/relay/status with 200: pass the relayer's answer through, and answer on its behalf with available: false when it cannot, reporting RELAYER_NOT_CONFIGURED for a DAO without a relayer and RELAYER_UNREACHABLE when the relayer is down, times out, answers with anything but a valid status, or its circuit is open. The probe goes straight to the relayer, so it never trips or occupies the circuit breaker that real relay traffic depends on. It logs once when it starts answering for a relayer that cannot answer and once when that relayer is reachable again diff --git a/.changeset/relayer-gasless-status.md b/.changeset/relayer-gasless-status.md new file mode 100644 index 0000000000..a0a7e38b41 --- /dev/null +++ b/.changeset/relayer-gasless-status.md @@ -0,0 +1,5 @@ +--- +"@anticapture/relayer": minor +--- + +Add GET /relay/status, which answers whether gasless relaying can be used right now from every check the relayer can run on itself (pause switch, RPC reachability and chain, wallet balance, rate limiter, stuck pending transactions, repeated broadcast failures) and always answers 200 with the failed checks listed as reasons, and add the GASLESS_PAUSED kill switch, which refuses every relayed vote, delegation, queue and execute with 503 RELAYER_PAUSED before a signature is recovered or the chain is read, and keeps webhook-driven enactment from broadcasting. GASLESS_PAUSED accepts true/1/yes/on (any case) and refuses to start on a value it does not recognise, and the status is published on /metrics as relayer_gasless_available and relayer_gasless_unavailable_reason for the new GaslessRelayerUnavailable alert diff --git a/apps/dashboard/features/governance/components/modals/VotePendingContent.tsx b/apps/dashboard/features/governance/components/modals/VotePendingContent.tsx index f9f440462f..9443de24fc 100644 --- a/apps/dashboard/features/governance/components/modals/VotePendingContent.tsx +++ b/apps/dashboard/features/governance/components/modals/VotePendingContent.tsx @@ -1,23 +1,36 @@ -import { ExternalLink, Hourglass } from "lucide-react"; +import { ExternalLink, Hourglass, Wallet } from "lucide-react"; import type { Hash } from "viem"; import { InlineAlert } from "@/shared/components/design-system/alerts/inline-alert/InlineAlert"; +import { Button } from "@/shared/components/design-system/buttons/button/Button"; interface VotePendingContentProps { hash: Hash | null; explorerBaseUrl?: string; + /** + * Offered for a relayed vote the relayer never confirmed: checks the chain + * and, if no vote from this address is there and no relayed transaction is + * still waiting to be mined, hands the ballot to the wallet. Absent, the + * screen keeps its old wait-only behaviour. + */ + onVoteWithWallet?: () => void; + isCheckingChain?: boolean; } const shortenHash = (hash: string) => `${hash.slice(0, 10)}…${hash.slice(-8)}`; /** - * A vote that may be on-chain but could not be confirmed here. It offers no - * way to vote again: the wallet would sign the same ballot, and the relayer - * would only be asked to replay it. + * A vote that may be on-chain but could not be confirmed here. A relayed one + * may also never land: the relayer may not have sent it, or its transaction + * may be dropped or revert. So the voter gets a way to cast the vote from + * their own wallet rather than a screen that waits forever on a service that + * is not answering; the chain is read before that way opens. */ export const VotePendingContent = ({ hash, explorerBaseUrl, + onVoteWithWallet, + isCheckingChain = false, }: VotePendingContentProps) => (
@@ -43,10 +56,30 @@ export const VotePendingContent = ({ + + {onVoteWithWallet && ( + + )}
); diff --git a/apps/dashboard/features/governance/components/modals/VotingModal.tsx b/apps/dashboard/features/governance/components/modals/VotingModal.tsx index d57d50e540..b5f6e72cd5 100644 --- a/apps/dashboard/features/governance/components/modals/VotingModal.tsx +++ b/apps/dashboard/features/governance/components/modals/VotingModal.tsx @@ -1,14 +1,20 @@ "use client"; import { useQueryClient } from "@tanstack/react-query"; -import { Check, User2Icon, X } from "lucide-react"; +import { Check, User2Icon, Wallet, X, Zap } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import type { Address } from "viem"; import { formatUnits } from "viem"; import { useAccount, usePublicClient, useWalletClient } from "wagmi"; -import { getRateLimitQueryKey } from "@anticapture/client/hooks"; -import type { GetRateLimitPathParamsDaoEnumKey } from "@anticapture/client"; +import { + getRateLimitQueryKey, + getRelayerStatusQueryKey, +} from "@anticapture/client/hooks"; +import type { + GetRateLimitPathParamsDaoEnumKey, + GetRelayerStatusPathParamsDaoEnumKey, +} from "@anticapture/client"; import { LoadingComponent } from "@/features/governance/components/modals/LoadingContent"; import { VoteAlreadyCastContent } from "@/features/governance/components/modals/VoteAlreadyCastContent"; @@ -17,20 +23,34 @@ import { VoteSuccessContent } from "@/features/governance/components/modals/Vote import { VoteOption } from "@/features/governance/components/proposal-overview/VoteOption"; import { useVoteSubmission } from "@/features/governance/hooks/useVoteSubmission"; import type { ProposalDetails } from "@/features/governance/types"; -import { relayGaslessVote } from "@/features/governance/utils/relayGaslessVote"; +import { + hasVotedOnChain, + isRelayedVotePending, + relayGaslessVote, +} from "@/features/governance/utils/relayGaslessVote"; import { showCustomToast } from "@/features/governance/utils/showCustomToast"; import { readSubmission } from "@/features/governance/utils/submissionStore"; import { buildTornDelegatedVoteFrom } from "@/features/governance/utils/tornDelegatedVote"; import { voteOnProposal } from "@/features/governance/utils/voteOnProposal"; import { + canReturnToGasless, + getSubmitLabel, + getVotePayment, +} from "@/features/governance/utils/votePayment"; +import { + ALREADY_ON_CHAIN, canStartVote, claimVote, + confirmLandedVote, getVoteScreen, + releaseUnlandedVote, settlementForGaslessVote, settlementForWalletVote, settleVote, + updateWalletFallback, type VoteSettlement, } from "@/features/governance/utils/voteSubmission"; +import { InlineAlert } from "@/shared/components/design-system/alerts/inline-alert/InlineAlert"; import { BadgeStatus } from "@/shared/components/design-system/badges/badge-status/BadgeStatus"; import { Button } from "@/shared/components/design-system/buttons/button/Button"; import { @@ -41,12 +61,14 @@ import { useScreenSize } from "@/shared/hooks"; import { useGaslessEligibility, useRelayerConfig, + useRelayerStatus, } from "@/shared/hooks/useGaslessRelayer"; import { useDelegators } from "@/shared/hooks/useDelegators"; import daoConfigByDaoId from "@/shared/dao-config"; import { DaoIdEnum } from "@/shared/types/daos"; import { formatNumberUserReadable } from "@/shared/utils/formatNumberUserReadable"; import { + describeGaslessFailure, isUserRejection, mapRelayerError, } from "@/shared/utils/gaslessRelayerError"; @@ -91,6 +113,7 @@ export const VotingModal = ({ // changed the store offers the ballot again, so this keeps the success // screen up until the modal is closed. const [hasLandedHere, setHasLandedHere] = useState(false); + const [isCheckingChain, setIsCheckingChain] = useState(false); const { isMobile } = useScreenSize(); @@ -166,6 +189,7 @@ export const VotingModal = ({ key: voteKey, submission, details: voteDetails, + walletFallback: { gaslessFailure, prefersWallet }, } = useVoteSubmission(daoId, proposal?.id, address); const screen = hasLandedHere ? "landed" @@ -238,6 +262,36 @@ export const VotingModal = ({ const { minVotingPower } = useRelayerConfig(daoId); const { isEligible: isGaslessEligible, remaining: voteRemaining } = useGaslessEligibility(daoId, address, "vote"); + const daoHasRelayer = daoConfig.gaslessRelayer === true; + const { isAvailable: isRelayerAvailable } = useRelayerStatus(daoId); + const payment = getVotePayment({ + daoHasRelayer, + isRelayerAvailable, + isEligible: isGaslessEligible, + hasComment: comment.trim().length > 0, + gaslessFailed: gaslessFailure !== null, + prefersWallet, + }); + + // A relay that failed may have been the first sign of a broken relayer, so + // the next status read should not wait out the stale time. + const refreshRelayerStatus = () => { + void queryClient.invalidateQueries({ + queryKey: getRelayerStatusQueryKey( + daoId.toLowerCase() as GetRelayerStatusPathParamsDaoEnumKey, + ), + }); + }; + + // Whether the governor already holds a vote from this voter, read on the + // DAO's chain. Only where a vote cannot be changed: elsewhere a second vote + // is a legitimate change, not a duplicate. A failed read answers no, which + // leaves the voter on the path they chose. + const isAlreadyOnChain = async (): Promise => { + const governor = daoConfig.daoOverview.contracts.governor; + if (canChangeVote || !governor || !publicClient || !address) return false; + return hasVotedOnChain(publicClient, governor, proposal.id, address); + }; // Opening starts a fresh ballot only when a new submission may start. A // vote that is out, landed or in doubt keeps its screen: resetting it here @@ -296,12 +350,9 @@ export const VotingModal = ({ return; const voteNumber = VOTE_NUMBER[vote]; - // castVoteBySig carries no reason, so a vote with a comment goes through - // the wallet rather than dropping the comment. - const isGasless = - isGaslessEligible && - daoConfig.gaslessRelayer === true && - comment.trim().length === 0; + // getVotePayment sends a commented vote through the wallet, since + // castVoteBySig carries no reason, and every relayer problem as well. + const isGasless = payment.mode === "gasless"; if (isGasless && !publicClient) return; // Taken before the first await, so a double click or a reopened modal @@ -346,7 +397,15 @@ export const VotingModal = ({ "error", ); } + if (outcome.status === "unknown") refreshRelayerStatus(); } else { + // A relayed attempt from this voter may land after the relayer gave + // up on it. Reading the chain first keeps the wallet from paying for + // a vote the governor already holds. + if (gaslessFailure !== null && (await isAlreadyOnChain())) { + settlement = ALREADY_ON_CHAIN; + return; + } const receipt = await voteOnProposal( vote, proposal.id, @@ -359,6 +418,10 @@ export const VotingModal = ({ tornDelegatedVoteAddresses, ); if (receipt?.status === "reverted") { + if (await isAlreadyOnChain()) { + settlement = ALREADY_ON_CHAIN; + return; + } showCustomToast("Your vote transaction reverted on-chain.", "error"); } settlement = settlementForWalletVote( @@ -368,9 +431,25 @@ export const VotingModal = ({ } catch (error) { // relayGaslessVote rejects only when nothing was sent: a declined // signature, or a relayer refusal before broadcasting. Both are safe to - // retry, so the ballot comes back. + // retry, so the ballot comes back. A refusal moves that ballot to the + // wallet and says why, rather than offering the relayer that just + // refused it. console.error(error); - if (!isUserRejection(error)) { + if (isGasless && !isUserRejection(error)) { + updateWalletFallback(voteKey, { + gaslessFailure: describeGaslessFailure(error, { + operation: "vote", + minVotingPower, + decimals: daoConfig.decimals, + symbol: daoConfig.name, + }), + }); + refreshRelayerStatus(); + } else if (!isUserRejection(error) && (await isAlreadyOnChain())) { + // The wallet refused to send, or the send failed, because a vote is + // already there: a relayed one that landed first counts. + settlement = ALREADY_ON_CHAIN; + } else if (!isUserRejection(error)) { showCustomToast( mapRelayerError(error, { operation: "vote", @@ -393,6 +472,43 @@ export const VotingModal = ({ } }; + // The relayer never confirmed the vote, with or without a hash. The chain + // decides: a vote already there is the relayed one landing late, a relayed + // transaction still in the mempool is left to land, and anything else + // (never sent, dropped, reverted) hands the same ballot to the wallet. + const handleVoteWithWallet = async () => { + if (!voteKey || !address || !publicClient) return; + if (submission.kind !== "ambiguous") return; + setIsCheckingChain(true); + try { + if (await isAlreadyOnChain()) { + confirmLandedVote(voteKey); + return; + } + if ( + submission.hash && + (await isRelayedVotePending(publicClient, submission.hash)) + ) { + showCustomToast( + "The relayed vote is still waiting to be mined. Follow it on the explorer.", + "error", + ); + return; + } + if (releaseUnlandedVote(voteKey)) { + updateWalletFallback(voteKey, { + gaslessFailure: "The relayer did not confirm your vote.", + }); + refreshRelayerStatus(); + } + } finally { + setIsCheckingChain(false); + } + }; + + const canFallBackFromPending = + submission.kind === "ambiguous" && submission.mode === "gasless"; + const submitDisabled = !address || !chain || @@ -436,6 +552,10 @@ export const VotingModal = ({ ) : screen === "in-flight" ? ( setComment(e.target.value)} /> - {isGaslessEligible && comment.trim().length > 0 && ( + {payment.mode === "wallet" && payment.reason === "comment" && (

Adding a comment requires a wallet transaction (gas fee). Votes without comments are gasless.

)}
+ + {/* How the vote is paid for */} + {daoHasRelayer && ( +
+ {payment.mode === "gasless" && ( +
+

+ + Gas-free vote: the relayer pays the network fee. +

+ +
+ )} + {payment.mode === "wallet" && + payment.reason === "relayer-unavailable" && ( + + )} + {payment.mode === "wallet" && + payment.reason === "gasless-failed" && ( + + )} + {canReturnToGasless(payment) && ( +
+

+ + You pay the network fee from your wallet. +

+ +
+ )} +
+ )} )} @@ -557,11 +739,15 @@ export const VotingModal = ({ )} diff --git a/apps/dashboard/features/governance/hooks/useVoteSubmission.ts b/apps/dashboard/features/governance/hooks/useVoteSubmission.ts index 5eece0e071..877d35da45 100644 --- a/apps/dashboard/features/governance/hooks/useVoteSubmission.ts +++ b/apps/dashboard/features/governance/hooks/useVoteSubmission.ts @@ -11,8 +11,11 @@ import { } from "@/features/governance/utils/submissionStore"; import { readVoteDetails, + readWalletFallback, + subscribeToWalletFallback, voteSubmissionKey, type VoteDetails, + type WalletFallback, } from "@/features/governance/utils/voteSubmission"; interface UseVoteSubmissionResult { @@ -20,6 +23,7 @@ interface UseVoteSubmissionResult { key: string | null; submission: SubmissionState; details: VoteDetails; + walletFallback: WalletFallback; } /** @@ -46,11 +50,19 @@ export const useVoteSubmission = ( ); const submission = useSyncExternalStore(subscribe, read, read); + const readFallback = useCallback(() => readWalletFallback(key), [key]); + const walletFallback = useSyncExternalStore( + subscribeToWalletFallback, + readFallback, + readFallback, + ); + return { key, submission, details: key ? readVoteDetails(key) : { support: null, alreadyVoted: false }, + walletFallback, }; }; diff --git a/apps/dashboard/features/governance/utils/relayGaslessVote.test.ts b/apps/dashboard/features/governance/utils/relayGaslessVote.test.ts index 0bbabd0168..befc5d802b 100644 --- a/apps/dashboard/features/governance/utils/relayGaslessVote.test.ts +++ b/apps/dashboard/features/governance/utils/relayGaslessVote.test.ts @@ -3,6 +3,7 @@ import type { Account, Address, Hash, Hex } from "viem"; import { relayVote } from "@anticapture/client"; import { + isRelayedVotePending, relayGaslessVote, type BallotSigner, type VoteChainReader, @@ -248,3 +249,47 @@ describe("relayGaslessVote", () => { expect(signTypedData).toHaveBeenCalledTimes(1); }); }); + +describe("isRelayedVotePending", () => { + const withTransaction = (getTransaction: jest.Mock) => + ({ getTransaction }) as unknown as Parameters< + typeof isRelayedVotePending + >[0]; + + beforeEach(() => { + jest.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it("holds the wallet back while the relayed transaction waits in the mempool", async () => { + const client = withTransaction( + jest.fn().mockResolvedValue({ blockNumber: null }), + ); + await expect(isRelayedVotePending(client, transactionHash)).resolves.toBe( + true, + ); + }); + + it("does not for a mined transaction: hasVoted answers that one", async () => { + const client = withTransaction( + jest.fn().mockResolvedValue({ blockNumber: BigInt(21_000_000) }), + ); + await expect(isRelayedVotePending(client, transactionHash)).resolves.toBe( + false, + ); + }); + + // A dropped transaction is unknown to the node, and a failed read says + // nothing; neither may keep the voter from their wallet. + it("does not for a transaction the node cannot find or a failed read", async () => { + const client = withTransaction( + jest.fn().mockRejectedValue(new Error("Transaction not found")), + ); + await expect(isRelayedVotePending(client, transactionHash)).resolves.toBe( + false, + ); + }); +}); diff --git a/apps/dashboard/features/governance/utils/relayGaslessVote.ts b/apps/dashboard/features/governance/utils/relayGaslessVote.ts index d02a9ac583..b40113aec2 100644 --- a/apps/dashboard/features/governance/utils/relayGaslessVote.ts +++ b/apps/dashboard/features/governance/utils/relayGaslessVote.ts @@ -80,9 +80,10 @@ type RelayGaslessVoteParams = { /** * A read that cannot run says nothing, so it lets the vote through: the - * relayer simulates before sending and refuses a duplicate on its own. + * relayer simulates before sending and refuses a duplicate on its own, and + * on the wallet fallback the governor refuses it just the same. */ -const hasVotedOnChain = async ( +export const hasVotedOnChain = async ( publicClient: VoteChainReader, governor: Address, proposalId: string, @@ -101,6 +102,26 @@ const hasVotedOnChain = async ( } }; +/** + * Whether a relayed transaction is still waiting to be mined. Only a node + * that knows the transaction and has no block for it says yes: a mined one + * is answered by `hasVoted`, and one the node has never seen or has dropped + * will not land. A read that fails says nothing, so it does not hold the + * voter back from their wallet. + */ +export const isRelayedVotePending = async ( + publicClient: Pick, + hash: Hash, +): Promise => { + try { + const transaction = await publicClient.getTransaction({ hash }); + return transaction.blockNumber === null; + } catch (error) { + console.error(error); + return false; + } +}; + /** * Casts a vote through the relayer, which pays the gas, with exactly one * relay call per run. diff --git a/apps/dashboard/features/governance/utils/votePayment.test.ts b/apps/dashboard/features/governance/utils/votePayment.test.ts new file mode 100644 index 0000000000..3565540709 --- /dev/null +++ b/apps/dashboard/features/governance/utils/votePayment.test.ts @@ -0,0 +1,142 @@ +/** + * The one rule these tests hold the ballot to: whatever the relayer is doing, + * a voter who can vote is offered a way to, and the wallet is that way + * whenever the relayer cannot be trusted with the ballot. + */ + +import { + canReturnToGasless, + getSubmitLabel, + getVotePayment, +} from "@/features/governance/utils/votePayment"; + +const healthy = { + daoHasRelayer: true, + isRelayerAvailable: true, + isEligible: true, + hasComment: false, + gaslessFailed: false, + prefersWallet: false, +} as const; + +describe("getVotePayment", () => { + it("relays a plain vote when the relayer is available and the voter eligible", () => { + expect(getVotePayment(healthy)).toEqual({ mode: "gasless" }); + }); + + it("uses the wallet, silently, on a DAO without a relayer", () => { + expect(getVotePayment({ ...healthy, daoHasRelayer: false })).toEqual({ + mode: "wallet", + reason: "none", + }); + }); + + it("uses the wallet and says so when the relayer reports itself unavailable", () => { + expect(getVotePayment({ ...healthy, isRelayerAvailable: false })).toEqual({ + mode: "wallet", + reason: "relayer-unavailable", + }); + }); + + // Eligibility is computed from the status too, so an unavailable relayer + // also reads as "not eligible"; the voter must still hear the real reason. + it("names the unavailable relayer even when eligibility follows it down", () => { + expect( + getVotePayment({ + ...healthy, + isRelayerAvailable: false, + isEligible: false, + }), + ).toEqual({ mode: "wallet", reason: "relayer-unavailable" }); + }); + + it("does not hold the ballot while the status is loading", () => { + expect(getVotePayment({ ...healthy, isRelayerAvailable: null })).toEqual({ + mode: "wallet", + reason: "none", + }); + }); + + it("uses the wallet without a warning for a voter the relayer would not sponsor", () => { + expect(getVotePayment({ ...healthy, isEligible: false })).toEqual({ + mode: "wallet", + reason: "none", + }); + }); + + it("keeps a failed relay on the wallet even if the status still says available", () => { + expect(getVotePayment({ ...healthy, gaslessFailed: true })).toEqual({ + mode: "wallet", + reason: "gasless-failed", + }); + }); + + it("sends a commented vote through the wallet", () => { + expect(getVotePayment({ ...healthy, hasComment: true })).toEqual({ + mode: "wallet", + reason: "comment", + }); + }); + + it("honours a voter who chose to pay gas", () => { + expect(getVotePayment({ ...healthy, prefersWallet: true })).toEqual({ + mode: "wallet", + reason: "wallet-chosen", + }); + }); + + it("never answers anything but gasless or wallet", () => { + const flags = [true, false]; + for (const daoHasRelayer of flags) + for (const isRelayerAvailable of [true, false, null]) + for (const isEligible of flags) + for (const hasComment of flags) + for (const gaslessFailed of flags) + for (const prefersWallet of flags) { + const payment = getVotePayment({ + daoHasRelayer, + isRelayerAvailable, + isEligible, + hasComment, + gaslessFailed, + prefersWallet, + }); + if (payment.mode === "gasless") { + expect(isRelayerAvailable).toBe(true); + expect(gaslessFailed).toBe(false); + } else { + expect(payment.mode).toBe("wallet"); + } + } + }); +}); + +describe("canReturnToGasless", () => { + it("offers the relayer back only to a voter who left it by choice", () => { + expect( + canReturnToGasless({ mode: "wallet", reason: "wallet-chosen" }), + ).toBe(true); + expect( + canReturnToGasless({ mode: "wallet", reason: "gasless-failed" }), + ).toBe(false); + expect( + canReturnToGasless({ mode: "wallet", reason: "relayer-unavailable" }), + ).toBe(false); + expect(canReturnToGasless({ mode: "gasless" })).toBe(false); + }); +}); + +describe("getSubmitLabel", () => { + it("names the wallet on a DAO that also offers gas-free votes", () => { + expect( + getSubmitLabel({ mode: "wallet", reason: "relayer-unavailable" }, true), + ).toBe("Vote with wallet"); + expect(getSubmitLabel({ mode: "gasless" }, true)).toBe("Submit gas-free"); + }); + + it("keeps the plain label on a DAO without a relayer", () => { + expect(getSubmitLabel({ mode: "wallet", reason: "none" }, false)).toBe( + "Submit", + ); + }); +}); diff --git a/apps/dashboard/features/governance/utils/votePayment.ts b/apps/dashboard/features/governance/utils/votePayment.ts new file mode 100644 index 0000000000..9d838e4016 --- /dev/null +++ b/apps/dashboard/features/governance/utils/votePayment.ts @@ -0,0 +1,86 @@ +/** + * Who pays for a vote, and what the ballot says about it. + * + * The relayer is a convenience and the wallet is the guarantee: every state + * in which the relayer cannot be trusted with this ballot resolves to the + * wallet, and the ballot always says why, so a voter is never left with a + * Submit button that keeps failing on the same broken path. + * + * - "relayer-unavailable": the status endpoint did not answer `available: + * true` (paused, unfunded, RPC down, relayer unreachable, ...). + * - "gasless-failed": a relay attempt from this ballot was refused before + * anything was broadcast, or the relayer never confirmed it and the chain + * shows no vote. The relayer may be fine for others, but this voter has + * seen it fail, so offering it again would only fail again. + * - "wallet-chosen": the voter opted out of the relayer themselves. + * - "comment": `castVoteBySig` carries no reason, so a comment needs the + * wallet's `castVoteWithReason`. + * - "none": the DAO has no relayer, or this voter is not eligible for it + * (voting power, monthly limit). Nothing went wrong, so nothing is said. + */ +export type WalletVoteReason = + | "relayer-unavailable" + | "gasless-failed" + | "wallet-chosen" + | "comment" + | "none"; + +export type VotePayment = + | { mode: "gasless" } + | { mode: "wallet"; reason: WalletVoteReason }; + +interface VotePaymentInput { + /** The DAO is configured with a gasless relayer at all. */ + daoHasRelayer: boolean; + /** The relayer status answer: null while it loads. */ + isRelayerAvailable: boolean | null; + /** Voting power and monthly limit allow a relayed vote for this voter. */ + isEligible: boolean; + hasComment: boolean; + gaslessFailed: boolean; + prefersWallet: boolean; +} + +export const getVotePayment = ({ + daoHasRelayer, + isRelayerAvailable, + isEligible, + hasComment, + gaslessFailed, + prefersWallet, +}: VotePaymentInput): VotePayment => { + if (!daoHasRelayer) return { mode: "wallet", reason: "none" }; + if (gaslessFailed) return { mode: "wallet", reason: "gasless-failed" }; + if (isRelayerAvailable === false) { + return { mode: "wallet", reason: "relayer-unavailable" }; + } + // Still loading, or not eligible: the wallet works either way, and holding + // the ballot until the relayer answers would make voting depend on it. + if (isRelayerAvailable === null || !isEligible) { + return { mode: "wallet", reason: "none" }; + } + if (hasComment) return { mode: "wallet", reason: "comment" }; + if (prefersWallet) return { mode: "wallet", reason: "wallet-chosen" }; + return { mode: "gasless" }; +}; + +/** + * Whether the voter can switch back to the relayer from where they are: only + * when they left it by choice. A failed or unavailable relayer is not + * offered again from the same ballot. + */ +export const canReturnToGasless = (payment: VotePayment): boolean => + payment.mode === "wallet" && payment.reason === "wallet-chosen"; + +/** + * The Submit label. A DAO with a relayer has two ways to vote, so the button + * names the one it will use whenever it is the wallet; a DAO without one + * keeps the plain label it always had. + */ +export const getSubmitLabel = ( + payment: VotePayment, + daoHasRelayer: boolean, +): string => { + if (payment.mode === "gasless") return "Submit gas-free"; + return daoHasRelayer ? "Vote with wallet" : "Submit"; +}; diff --git a/apps/dashboard/features/governance/utils/voteSubmission.test.ts b/apps/dashboard/features/governance/utils/voteSubmission.test.ts index 864dc322ad..e94d7403a2 100644 --- a/apps/dashboard/features/governance/utils/voteSubmission.test.ts +++ b/apps/dashboard/features/governance/utils/voteSubmission.test.ts @@ -16,12 +16,18 @@ import { readSubmission } from "@/features/governance/utils/submissionStore"; import { claimVote, getVoteScreen, + ALREADY_ON_CHAIN, + confirmLandedVote, isSubmittedVoteIndexed, readVoteDetails, + readWalletFallback, + releaseUnlandedVote, settleVote, settlementForGaslessVote, settlementForWalletVote, submittedVoteSupport, + subscribeToWalletFallback, + updateWalletFallback, voteSubmissionKey, } from "@/features/governance/utils/voteSubmission"; import { DaoIdEnum } from "@/shared/types/daos"; @@ -248,3 +254,139 @@ describe("isSubmittedVoteIndexed", () => { ).toBe(true); }); }); + +describe("releaseUnlandedVote", () => { + it("frees a relayed vote that came back with no hash", () => { + const key = freshKey(); + claimVote(key, "gasless", 1); + settleVote( + key, + settlementForGaslessVote({ status: "unknown", hash: null }), + ); + expect(getVoteScreen(readSubmission(key), fixedVote)).toBe("pending"); + + expect(releaseUnlandedVote(key)).toBe(true); + expect(getVoteScreen(readSubmission(key), fixedVote)).toBe("ballot"); + expect(claimVote(key, "wallet", 1)).toBe(true); + }); + + // A hash proves the relayer sent it, not that it lands: a dropped or + // reverted relay must not strand the voter. The caller checks the chain. + it("frees a relayed vote whose hash never confirmed, keeping the hash", () => { + const key = freshKey(); + claimVote(key, "gasless", 1); + settleVote(key, settlementForGaslessVote({ status: "unconfirmed", hash })); + + expect(releaseUnlandedVote(key)).toBe(true); + expect(readSubmission(key)).toEqual({ + kind: "done", + mode: "gasless", + outcome: "failed", + hash, + }); + expect(getVoteScreen(readSubmission(key), fixedVote)).toBe("ballot"); + }); + + it("leaves every other state alone", () => { + const inFlight = freshKey(); + claimVote(inFlight, "gasless", 1); + expect(releaseUnlandedVote(inFlight)).toBe(false); + expect(readSubmission(inFlight).kind).toBe("in-flight"); + + const landedKey = freshKey(); + claimVote(landedKey, "gasless", 1); + settleVote( + landedKey, + settlementForGaslessVote({ status: "success", hash }), + ); + expect(releaseUnlandedVote(landedKey)).toBe(false); + expect(readSubmission(landedKey)).toMatchObject({ outcome: "landed" }); + + expect(releaseUnlandedVote(freshKey())).toBe(false); + }); +}); + +describe("confirmLandedVote", () => { + it("turns a vote in doubt into a landed one once the chain shows it", () => { + const key = freshKey(); + claimVote(key, "gasless", 1); + settleVote(key, settlementForGaslessVote({ status: "unconfirmed", hash })); + + expect(confirmLandedVote(key)).toBe(true); + expect(getVoteScreen(readSubmission(key), fixedVote)).toBe("landed"); + expect(isSubmittedVoteIndexed(readSubmission(key), undefined)).toBe(false); + expect( + isSubmittedVoteIndexed(readSubmission(key), { transactionHash: hash }), + ).toBe(true); + }); + + it("does nothing outside a vote in doubt", () => { + const key = freshKey(); + claimVote(key, "gasless", 1); + expect(confirmLandedVote(key)).toBe(false); + expect(readSubmission(key).kind).toBe("in-flight"); + }); +}); + +describe("ALREADY_ON_CHAIN", () => { + // A wallet vote that found the relayed one already counted. + it("shows the vote on record and never offers the ballot again", () => { + const key = freshKey(); + claimVote(key, "wallet", 0); + settleVote(key, ALREADY_ON_CHAIN); + + expect(readVoteDetails(key)).toEqual({ support: null, alreadyVoted: true }); + expect( + getVoteScreen(readSubmission(key), { + canChangeVote: false, + alreadyVoted: readVoteDetails(key).alreadyVoted, + }), + ).toBe("already-voted"); + expect(claimVote(key, "wallet", 0)).toBe(false); + }); +}); + +describe("wallet fallback", () => { + it("follows the voter, not the modal: another account starts clean", () => { + const first = voteSubmissionKey(DaoIdEnum.ENS, "fallback-1", voter); + const second = voteSubmissionKey( + DaoIdEnum.ENS, + "fallback-1", + "0x983110309620D911731Ac0932219af06091b6744", + ); + + updateWalletFallback(first, { + gaslessFailure: "The relayer is out of funds.", + }); + + expect(readWalletFallback(first)).toEqual({ + gaslessFailure: "The relayer is out of funds.", + prefersWallet: false, + }); + expect(readWalletFallback(second)).toEqual({ + gaslessFailure: null, + prefersWallet: false, + }); + expect(readWalletFallback(null)).toEqual({ + gaslessFailure: null, + prefersWallet: false, + }); + }); + + it("merges updates and notifies subscribers", () => { + const key = voteSubmissionKey(DaoIdEnum.ENS, "fallback-2", voter); + const onChange = jest.fn(); + const unsubscribe = subscribeToWalletFallback(onChange); + + updateWalletFallback(key, { prefersWallet: true }); + updateWalletFallback(key, { gaslessFailure: "Gas-free voting is paused." }); + unsubscribe(); + updateWalletFallback(key, { prefersWallet: false }); + + expect(onChange).toHaveBeenCalledTimes(2); + expect(readWalletFallback(key)).toEqual({ + gaslessFailure: "Gas-free voting is paused.", + prefersWallet: false, + }); + }); +}); diff --git a/apps/dashboard/features/governance/utils/voteSubmission.ts b/apps/dashboard/features/governance/utils/voteSubmission.ts index 8d0ecd41f9..3a09caa4ac 100644 --- a/apps/dashboard/features/governance/utils/voteSubmission.ts +++ b/apps/dashboard/features/governance/utils/voteSubmission.ts @@ -48,6 +48,48 @@ const detailsByKey = new Map(); export const readVoteDetails = (key: string): VoteDetails => detailsByKey.get(key) ?? NO_VOTE_DETAILS; +/** + * How this voter's ballot is to be paid for, beyond what the relayer status + * says. Keyed like the submission, so it follows the voter rather than the + * modal: a second account in the same tab starts clean, and a remount of the + * proposal page does not offer the relayer that just refused this voter. + * + * - `gaslessFailure`: why the last relayed attempt did not go through. Once + * set, the ballot stays on the wallet for the life of the page, since the + * same relayer would most likely refuse the same voter the same way. + * - `prefersWallet`: the voter chose to pay gas while the relayer was on offer. + */ +export interface WalletFallback { + gaslessFailure: string | null; + prefersWallet: boolean; +} + +const NO_WALLET_FALLBACK: WalletFallback = { + gaslessFailure: null, + prefersWallet: false, +}; + +const fallbackByKey = new Map(); +const fallbackListeners = new Set<() => void>(); + +export const readWalletFallback = (key: string | null): WalletFallback => + (key ? fallbackByKey.get(key) : undefined) ?? NO_WALLET_FALLBACK; + +export const updateWalletFallback = ( + key: string, + patch: Partial, +): void => { + fallbackByKey.set(key, { ...readWalletFallback(key), ...patch }); + for (const listener of fallbackListeners) listener(); +}; + +export const subscribeToWalletFallback = (onChange: () => void) => { + fallbackListeners.add(onChange); + return () => { + fallbackListeners.delete(onChange); + }; +}; + /** * Whether a new ballot may be submitted. A vote that is out or in doubt never * allows one. A landed vote allows one only where the governor lets a voter @@ -135,6 +177,65 @@ export const settlementForGaslessVote = ( } }; +/** + * Frees a relayed vote the relayer never confirmed, once the chain shows no + * vote from this address, so the voter can cast it from their own wallet. + * + * The caller reads `hasVoted` first and, when the relayer returned a hash, + * checks that the transaction is not still waiting in the mempool: a hash + * proves the relayer sent it, not that it will land, and a dropped or + * reverted one would otherwise strand the voter on a screen that waits for + * it forever. The relayer may still broadcast late after this; the worst + * that leaves is two transactions for one ballot, and the governor counts + * only the first. The second reverts and costs its sender gas, which is + * cheaper than a voter who cannot vote, and the wallet path reads the chain + * again before and after it sends. + */ +export const releaseUnlandedVote = (key: string): boolean => { + const current = readSubmission(key); + if (current.kind !== "ambiguous" || current.mode !== "gasless") { + return false; + } + writeSubmission(key, { + kind: "done", + mode: "gasless", + outcome: "failed", + hash: current.hash, + }); + return true; +}; + +/** + * Settles a vote whose outcome was in doubt once the chain shows a vote from + * this address: the relayed one landed after all. The submission becomes a + * landed vote, so the screen says so and the page keeps watching for it to + * be indexed, instead of offering the voter another way to cast it. + */ +export const confirmLandedVote = (key: string): boolean => { + const current = readSubmission(key); + if (current.kind !== "ambiguous") return false; + writeSubmission(key, { + kind: "done", + mode: current.mode, + outcome: "landed", + hash: current.hash, + }); + return true; +}; + +/** + * A wallet vote that found the governor already holding a vote from this + * address: an earlier relayed one landed first. That vote is the one that + * counts, so the run settles as landed and the screen says the vote was + * already cast, rather than reporting a failure and offering the ballot again. + */ +export const ALREADY_ON_CHAIN: VoteSettlement = { + kind: "done", + outcome: "landed", + hash: null, + alreadyVoted: true, +}; + /** The wallet path reports its confirmed hash, or null when nothing landed. */ export const settlementForWalletVote = (hash: Hash | null): VoteSettlement => hash diff --git a/apps/dashboard/features/holders-and-delegates/delegate/DelegationModal.tsx b/apps/dashboard/features/holders-and-delegates/delegate/DelegationModal.tsx index fc7fd5d896..57e9403c36 100644 --- a/apps/dashboard/features/holders-and-delegates/delegate/DelegationModal.tsx +++ b/apps/dashboard/features/holders-and-delegates/delegate/DelegationModal.tsx @@ -1,22 +1,29 @@ "use client"; import { useQueryClient } from "@tanstack/react-query"; -import { Check, Hourglass, PenLine } from "lucide-react"; -import { useCallback, useEffect, useState } from "react"; +import { Check, Hourglass, PenLine, Wallet } from "lucide-react"; +import { useCallback, useEffect, useRef, useState } from "react"; import type { Address } from "viem"; import { formatUnits } from "viem"; import { useAccount, useReadContract, useWalletClient } from "wagmi"; -import { getRateLimitQueryKey } from "@anticapture/client/hooks"; -import type { GetRateLimitPathParamsDaoEnumKey } from "@anticapture/client"; +import { + getRateLimitQueryKey, + getRelayerStatusQueryKey, +} from "@anticapture/client/hooks"; +import type { + GetRateLimitPathParamsDaoEnumKey, + GetRelayerStatusPathParamsDaoEnumKey, +} from "@anticapture/client"; import { delegateTo } from "@/features/holders-and-delegates/delegate/utils/delegateTo"; import { showCustomToast } from "@/features/governance/utils/showCustomToast"; import { + describeGaslessFailure, isUserRejection, - mapRelayerError, } from "@/shared/utils/gaslessRelayerError"; import { InlineAlert } from "@/shared/components/design-system/alerts/inline-alert/InlineAlert"; +import { Button } from "@/shared/components/design-system/buttons/button/Button"; import { EnsAvatar } from "@/shared/components/design-system/avatars/ens-avatar/EnsAvatar"; import { Modal } from "@/shared/components/design-system/modal/Modal"; import { SpinIcon } from "@/shared/components/icons/SpinIcon"; @@ -61,6 +68,22 @@ export const DelegationModal = ({ const queryClient = useQueryClient(); const [step, setStep] = useState("waiting-signature"); const [error, setError] = useState(null); + // The account whose gas-free delegation the relayer last refused or never + // confirmed. That account delegates from its own wallet from then on, the + // same rule the ballot follows: the wallet is the guarantee, and offering + // the relayer that just failed would only fail again. Keyed by account so + // switching wallets starts clean. + const [gaslessFailedFor, setGaslessFailedFor] = useState
( + null, + ); + const gaslessFailed = + !!userAddress && + gaslessFailedFor?.toLowerCase() === userAddress.toLowerCase(); + // One delegation at a time. The effect below runs handleDelegate whenever + // its inputs change while the modal waits for a signature, and the relayer + // status now refreshes in the background, so without this a refresh during + // signing would ask the wallet for a second signature. + const inFlight = useRef(false); const daoConfig = daoConfigByDaoId[daoId]; const tokenAddress = daoConfig?.daoOverview?.contracts?.token as @@ -90,11 +113,14 @@ export const DelegationModal = ({ const handleDelegate = useCallback(async () => { if (!userAddress || !walletClient || !tokenAddress) return; if (step === "pending-tx" || step === "success") return; + if (inFlight.current) return; + inFlight.current = true; setError(null); setStep("waiting-signature"); - const isGasless = isGaslessEligible && daoConfig?.gaslessRelayer === true; + const isGasless = + isGaslessEligible && daoConfig?.gaslessRelayer === true && !gaslessFailed; try { await delegateTo( tokenAddress, @@ -104,7 +130,7 @@ export const DelegationModal = ({ () => setStep("pending-tx"), chain, daoId, - isGaslessEligible, + isGasless, ); setStep("success"); showCustomToast("Delegation successful!", "success"); @@ -117,14 +143,23 @@ export const DelegationModal = ({ } if (isGasless) { - const message = mapRelayerError(err, { - operation: "delegate", - minVotingPower, - decimals, - symbol: daoConfig.name, + // Any relayer failure moves this account to its wallet. Delegating + // again to the same delegate is idempotent, so even a relayed + // delegation that lands late costs the wallet path nothing but gas. + setGaslessFailedFor(userAddress); + setError( + `${describeGaslessFailure(err, { + operation: "delegate", + minVotingPower, + decimals, + symbol: daoConfig.name, + })} You can still delegate with your wallet and pay the network fee.`, + ); + void queryClient.invalidateQueries({ + queryKey: getRelayerStatusQueryKey( + daoId.toLowerCase() as GetRelayerStatusPathParamsDaoEnumKey, + ), }); - showCustomToast(message, "error"); - setError(message); } else { const message = err instanceof Error ? err.message : "Delegation failed."; @@ -132,6 +167,7 @@ export const DelegationModal = ({ } setStep("error"); } finally { + inFlight.current = false; // The relayer counts each delegation against a monthly allowance, and // the cached count is stale for a minute: without this, a delegation // right after the last free one still goes to the relayer and gets 429. @@ -157,6 +193,7 @@ export const DelegationModal = ({ decimals, minVotingPower, isGaslessEligible, + gaslessFailed, queryClient, ]); @@ -196,19 +233,21 @@ export const DelegationModal = ({ showCopyAddress={false} /> - {isGaslessEligible && delegationRemaining !== null && ( - - )} + {isGaslessEligible && + !gaslessFailed && + delegationRemaining !== null && ( + + )} {/* Stepper */} @@ -230,6 +269,23 @@ export const DelegationModal = ({ label="Wait for the delegation to complete" /> + + {step === "error" && gaslessFailed && ( +
+ +
+ )} ); }; diff --git a/apps/dashboard/shared/hooks/useGaslessRelayer.ts b/apps/dashboard/shared/hooks/useGaslessRelayer.ts index 79ac9c871b..3176d0131a 100644 --- a/apps/dashboard/shared/hooks/useGaslessRelayer.ts +++ b/apps/dashboard/shared/hooks/useGaslessRelayer.ts @@ -5,14 +5,15 @@ import { useAccountBalanceByAccountId, useGetConfig, useGetRateLimit, - useGetRelayerBalance, + useGetRelayerStatus, useVotingPowerByAccountId, } from "@anticapture/client/hooks"; import type { AccountBalanceByAccountIdPathParamsDaoEnumKey, GetConfigPathParamsDaoEnumKey, GetRateLimitPathParamsDaoEnumKey, - GetRelayerBalancePathParamsDaoEnumKey, + GetRelayerStatusPathParamsDaoEnumKey, + RelayerStatusReason, VotingPowerByAccountIdPathParamsDaoEnumKey, } from "@anticapture/client"; @@ -30,33 +31,76 @@ interface UseRelayerConfigResult { const toDaoKey = (daoId: DaoIdEnum) => daoId.toLowerCase(); /** - * The relayer's balance, config and an address's rate limit change at most - * once per relayed transaction. The balance hook alone is mounted by the - * config, rate-limit and eligibility hooks and by every enactment modal, and - * with no stale time each new observer refetched: the relayer logged the same - * three GETs two or three times within one second. A vote invalidates the - * rate limit itself once it is relayed. + * The relayer's config and an address's rate limit change at most once per + * relayed transaction. The status hook alone is mounted by the config, + * rate-limit and eligibility hooks and by every enactment modal, and with no + * stale time each new observer refetched: the relayer logged the same three + * GETs two or three times within one second. A vote invalidates the rate + * limit itself once it is relayed. */ const RELAYER_QUERY_STALE_MS = 60_000; -interface UseRelayerBalanceResult { - hasEnoughBalance: boolean | null; +/** + * How long a relayer status answer is trusted. Shorter than the config and + * rate-limit stale time: the status is what decides whether the ballot goes + * through the relayer at all, and a relayer that broke a minute ago should + * stop being offered well before the voter gives up on it. + */ +const RELAYER_STATUS_STALE_MS = 30_000; + +/** + * How often an open page asks again. Every observer of the status lives as + * long as the page (the voting and delegation modals stay mounted while + * closed), so the stale time alone never triggers a refetch: without this a + * relayer that broke after the page loaded kept being offered until the + * window lost and regained focus. Paused while the tab is in the background. + */ +const RELAYER_STATUS_REFRESH_MS = 60_000; + +interface UseRelayerStatusResult { + /** + * Whether the gasless path may be offered. Null while the first answer is + * on its way; false for a DAO without a relayer, for every reason the + * relayer or the gateway reports, and for a request that failed outright. + */ + isAvailable: boolean | null; + reasons: RelayerStatusReason[]; isLoading: boolean; } -export const useRelayerBalance = ( - daoId: DaoIdEnum, -): UseRelayerBalanceResult => { +/** + * Whether the relayer can sponsor a transaction right now, from the one + * endpoint that checks everything it depends on: the pause switch, the RPC, + * the wallet balance, the rate limiter, stuck nonces and recent broadcast + * failures. Gateful answers on the relayer's behalf when it is down, so the + * gasless path is offered only on an explicit `available: true`. Anything + * else, including an error, leaves the voter on their own wallet, which is + * the path that never depends on this service. + */ +export const useRelayerStatus = (daoId: DaoIdEnum): UseRelayerStatusResult => { const enabled = daoConfigByDaoId[daoId].gaslessRelayer === true; - const { data, isLoading } = useGetRelayerBalance( - toDaoKey(daoId) as GetRelayerBalancePathParamsDaoEnumKey, - { query: { enabled, staleTime: RELAYER_QUERY_STALE_MS } }, + const { data, isLoading, isError } = useGetRelayerStatus( + toDaoKey(daoId) as GetRelayerStatusPathParamsDaoEnumKey, + { + query: { + enabled, + staleTime: RELAYER_STATUS_STALE_MS, + refetchInterval: RELAYER_STATUS_REFRESH_MS, + // A failed status read already means "use the wallet"; retrying it + // would only hold the ballot on the slower path for longer. + retry: false, + }, + }, ); + if (!enabled) return { isAvailable: false, reasons: [], isLoading: false }; + if (isError) return { isAvailable: false, reasons: [], isLoading: false }; + return { - hasEnoughBalance: data?.hasEnoughBalance ?? null, - isLoading: enabled && isLoading, + isAvailable: data ? data.available === true : null, + reasons: data?.reasons ?? [], + isLoading, }; }; @@ -69,25 +113,26 @@ interface UseGaslessEnactmentResult { /** * Queue and execute are permissionless and carry no signer, so the only * things that gate the gasless path are the DAO having a relayer and that - * relayer being funded. No voting power or per-address rate limit applies. + * relayer reporting itself available. No voting power or per-address rate + * limit applies. */ export const useGaslessEnactment = ( daoId: DaoIdEnum, ): UseGaslessEnactmentResult => { const gaslessEnabled = daoConfigByDaoId[daoId].gaslessRelayer === true; - const { hasEnoughBalance, isLoading } = useRelayerBalance(daoId); + const { isAvailable, isLoading } = useRelayerStatus(daoId); return { - isAvailable: gaslessEnabled && hasEnoughBalance === true, + isAvailable: gaslessEnabled && isAvailable === true, isLoading: gaslessEnabled && isLoading, }; }; export const useRelayerConfig = (daoId: DaoIdEnum): UseRelayerConfigResult => { const gaslessEnabled = daoConfigByDaoId[daoId].gaslessRelayer === true; - const { hasEnoughBalance, isLoading: balanceLoading } = - useRelayerBalance(daoId); - const enabled = gaslessEnabled && hasEnoughBalance === true; + const { isAvailable: isRelayerAvailable, isLoading: statusLoading } = + useRelayerStatus(daoId); + const enabled = gaslessEnabled && isRelayerAvailable === true; const { data, isLoading } = useGetConfig( toDaoKey(daoId) as GetConfigPathParamsDaoEnumKey, @@ -108,7 +153,7 @@ export const useRelayerConfig = (daoId: DaoIdEnum): UseRelayerConfigResult => { minVotingPower, voteLimit: data?.limits?.vote ?? null, delegationLimit: data?.limits?.delegation ?? null, - isLoading: gaslessEnabled && (balanceLoading || (enabled && isLoading)), + isLoading: gaslessEnabled && (statusLoading || (enabled && isLoading)), }; }; @@ -127,9 +172,9 @@ export const useRelayerRateLimit = ( ): UseRelayerRateLimitResult => { const gaslessEnabled = daoConfigByDaoId[daoId].gaslessRelayer === true && !!address; - const { hasEnoughBalance, isLoading: balanceLoading } = - useRelayerBalance(daoId); - const enabled = gaslessEnabled && hasEnoughBalance === true; + const { isAvailable: isRelayerAvailable, isLoading: statusLoading } = + useRelayerStatus(daoId); + const enabled = gaslessEnabled && isRelayerAvailable === true; const { data, isLoading } = useGetRateLimit( toDaoKey(daoId) as GetRateLimitPathParamsDaoEnumKey, @@ -143,7 +188,7 @@ export const useRelayerRateLimit = ( voteLimit: data?.vote?.limit ?? null, delegationLimit: data?.delegation?.limit ?? null, resetsAt: data?.resetsAt ?? null, - isLoading: gaslessEnabled && (balanceLoading || (enabled && isLoading)), + isLoading: gaslessEnabled && (statusLoading || (enabled && isLoading)), }; }; @@ -160,8 +205,8 @@ export const useGaslessEligibility = ( ): UseGaslessEligibilityResult => { const gaslessEnabled = daoConfigByDaoId[daoId].gaslessRelayer === true; - const { hasEnoughBalance, isLoading: balanceLoading } = - useRelayerBalance(daoId); + const { isAvailable: isRelayerAvailable, isLoading: statusLoading } = + useRelayerStatus(daoId); const { minVotingPower, isLoading: configLoading } = useRelayerConfig(daoId); const { voteRemaining, @@ -169,7 +214,8 @@ export const useGaslessEligibility = ( isLoading: rateLimitLoading, } = useRelayerRateLimit(daoId, address); - const queryEnabled = gaslessEnabled && !!address && hasEnoughBalance === true; + const queryEnabled = + gaslessEnabled && !!address && isRelayerAvailable === true; const isVote = operation === "vote"; @@ -197,14 +243,14 @@ export const useGaslessEligibility = ( const isLoading = gaslessEnabled && !!address && - (balanceLoading || - (hasEnoughBalance === true && + (statusLoading || + (isRelayerAvailable === true && (configLoading || rateLimitLoading || eligibilityLoading))); const isEligible = gaslessEnabled && !isLoading && - hasEnoughBalance === true && + isRelayerAvailable === true && rawEligibilityAmount !== null && minVotingPower !== null && BigInt(rawEligibilityAmount) >= minVotingPower && diff --git a/apps/dashboard/shared/services/posthog/ConditionalPostHog.tsx b/apps/dashboard/shared/services/posthog/ConditionalPostHog.tsx index 953ca8a2e1..d239c2d2ea 100644 --- a/apps/dashboard/shared/services/posthog/ConditionalPostHog.tsx +++ b/apps/dashboard/shared/services/posthog/ConditionalPostHog.tsx @@ -41,6 +41,11 @@ const ConditionalPostHog = () => { properties.dao = element.dataset.phDao; } + // Optionally add how an action is paid for (gasless or wallet) + if (element.dataset.phPayment) { + properties.payment = element.dataset.phPayment; + } + windowWithPostHog.posthog.capture( element.dataset.phEvent || "", properties, diff --git a/apps/dashboard/shared/utils/gaslessRelayerError.test.ts b/apps/dashboard/shared/utils/gaslessRelayerError.test.ts index 32d529a686..58cd999887 100644 --- a/apps/dashboard/shared/utils/gaslessRelayerError.test.ts +++ b/apps/dashboard/shared/utils/gaslessRelayerError.test.ts @@ -1,5 +1,6 @@ import { classifyRelayerFailure, + describeGaslessFailure, getRelayerErrorCode, getRelayerRevertedHash, mapRelayerEnactmentError, @@ -280,3 +281,81 @@ describe("getRelayerRevertedHash", () => { ).toBeNull(); }); }); + +describe("describeGaslessFailure", () => { + const vote = { + operation: "vote" as const, + minVotingPower: null, + decimals: 18, + symbol: "ENS", + }; + const delegate = { ...vote, operation: "delegate" as const }; + + it("names the relayer's own state rather than telling the user to wait", () => { + expect( + describeGaslessFailure(relayerError(503, "RELAYER_LOW_BALANCE"), vote), + ).toBe("The relayer is out of funds."); + expect( + describeGaslessFailure(relayerError(503, "RELAYER_PAUSED"), vote), + ).toBe("Gas-free voting is paused."); + expect( + describeGaslessFailure(relayerError(503, "RELAYER_PAUSED"), delegate), + ).toBe("Gas-free delegation is paused."); + }); + + // These sit right before "You can still vote with your wallet", so they + // must be about the gas-free allowance, not about voting itself. + it("frames the relayer's eligibility refusals as gas-free limits", () => { + expect( + describeGaslessFailure(relayerError(429, "RATE_LIMITED"), vote), + ).toBe("You've used all your gas-free operations for this month."); + expect( + describeGaslessFailure(relayerError(400, "INSUFFICIENT_VOTING_POWER"), { + ...vote, + minVotingPower: BigInt(10) ** BigInt(18), + }), + ).toBe("A gas-free vote needs at least 1 ENS."); + expect( + describeGaslessFailure( + relayerError(400, "INSUFFICIENT_VOTING_POWER"), + delegate, + ), + ).toBe("Your balance is below the minimum for a gas-free delegation."); + }); + + it("says nothing was sent only when the relayer refused before sending", () => { + expect( + describeGaslessFailure( + relayerError(409, "SIMULATION_FAILED", "reverted"), + vote, + ), + ).toBe("The gas-free vote did not go through, and nothing was sent."); + expect(describeGaslessFailure(relayerError(502), delegate)).toBe( + "The relayer did not confirm the gas-free delegation.", + ); + }); + + it("always answers one complete sentence and never says try again", () => { + for (const error of [ + relayerError(503), + relayerError(409, "SIMULATION_FAILED", "reverted"), + relayerError(429, "RATE_LIMITED"), + relayerError(400, "INSUFFICIENT_VOTING_POWER"), + new Error("network"), + ]) { + for (const context of [vote, delegate]) { + const text = describeGaslessFailure(error, context); + expect(text).toMatch(/\.$/); + expect(text).not.toMatch(/try again/i); + } + } + }); +}); + +describe("RELAYER_PAUSED", () => { + it("is a refusal before broadcasting, so the wallet path stays open", () => { + expect(classifyRelayerFailure(relayerError(503, "RELAYER_PAUSED"))).toBe( + "pre-broadcast", + ); + }); +}); diff --git a/apps/dashboard/shared/utils/gaslessRelayerError.ts b/apps/dashboard/shared/utils/gaslessRelayerError.ts index b06ad63a47..aeeccfd98f 100644 --- a/apps/dashboard/shared/utils/gaslessRelayerError.ts +++ b/apps/dashboard/shared/utils/gaslessRelayerError.ts @@ -64,10 +64,10 @@ export const getRelayerErrorCode = (error: unknown): string | undefined => /** * Every code the relayer can answer with before it signs anything, mirroring * `Errors` in `apps/relayer/src/errors.ts`. All of them mean no transaction - * was broadcast, so retrying is safe. The two 503s are in here for that - * reason: a low balance or an unreachable rate limiter are structured "did - * not send" answers, not the ambiguous 5xx they look like from the status - * alone. `TRANSACTION_REVERTED` is deliberately absent, being the one code + * was broadcast, so retrying is safe. The 503s are in here for that reason: + * a low balance, a paused relayer or an unreachable rate limiter are + * structured "did not send" answers, not the ambiguous 5xx they look like + * from the status alone. `TRANSACTION_REVERTED` is deliberately absent, being the one code * that reports a transaction the relayer did broadcast. */ const ENACTMENT_REJECTION_CODES = new Set([ @@ -75,6 +75,7 @@ const ENACTMENT_REJECTION_CODES = new Set([ "INVALID_SIGNATURE", "RATE_LIMITED", "RELAYER_LOW_BALANCE", + "RELAYER_PAUSED", "INVALID_CONTRACT", "RATE_LIMITER_UNAVAILABLE", "PROPOSAL_NOT_FOUND", @@ -166,6 +167,8 @@ const ENACTMENT_MESSAGES: Record< "The proposal data doesn't match what is on-chain, so the relayer refused to submit it.", RELAYER_LOW_BALANCE: () => "The relayer is out of funds right now. You can still complete this action with your own wallet.", + RELAYER_PAUSED: () => + "Gas-free transactions are paused right now. You can still complete this action with your own wallet.", }; export const mapRelayerEnactmentError = ( @@ -228,6 +231,48 @@ export const mapRelayerError = ( return GENERIC_MESSAGE; }; +/** + * Why a gas-free vote or delegation did not go through, as one complete + * sentence the modal puts right before "You can still ... with your wallet". + * It never says "try again later": the wallet is right there, so the copy + * points at it instead of at the relayer. The relayer's own refusals are + * about gas-free eligibility, not about voting at all, so they say so: "you + * don't have sufficient voting power" next to a wallet button would read as + * a contradiction. + */ +export const describeGaslessFailure = ( + error: unknown, + context: { + operation: "vote" | "delegate"; + minVotingPower: bigint | null; + decimals: number; + symbol: string; + }, +): string => { + const { code, status } = readRelayerError(error); + const noun = context.operation === "vote" ? "vote" : "delegation"; + + if (code === INSUFFICIENT_VOTING_POWER) { + return context.minVotingPower === null + ? `Your balance is below the minimum for a gas-free ${noun}.` + : `A gas-free ${noun} needs at least ${formatThreshold( + context.minVotingPower, + context.decimals, + context.symbol, + )}.`; + } + if (code === RATE_LIMITED || status === 429) { + return "You've used all your gas-free operations for this month."; + } + if (code === "RELAYER_LOW_BALANCE") return "The relayer is out of funds."; + if (code === "RELAYER_PAUSED") { + return `Gas-free ${context.operation === "vote" ? "voting" : "delegation"} is paused.`; + } + return classifyRelayerFailure(error) === "pre-broadcast" + ? `The gas-free ${noun} did not go through, and nothing was sent.` + : `The relayer did not confirm the gas-free ${noun}.`; +}; + export const isUserRejection = (error: unknown): boolean => { const message = error instanceof Error diff --git a/apps/dashboard/stories/GaslessVoteFallback.stories.tsx b/apps/dashboard/stories/GaslessVoteFallback.stories.tsx new file mode 100644 index 0000000000..9641ffec5a --- /dev/null +++ b/apps/dashboard/stories/GaslessVoteFallback.stories.tsx @@ -0,0 +1,394 @@ +import type { Meta, StoryObj } from "@storybook/nextjs"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { useState } from "react"; +import { expect, userEvent, waitFor, within } from "storybook/test"; +import { + decodeFunctionData, + encodeFunctionResult, + parseAbi, + type Address, +} from "viem"; +import { mainnet } from "viem/chains"; +import { createConfig, http, WagmiProvider } from "wagmi"; +import { mock } from "wagmi/connectors"; + +import type { + RelayerRateLimitResponse, + RelayerStatusResponse, +} from "@anticapture/client"; + +import { VotingModal } from "@/features/governance/components/modals/VotingModal"; +import { + ProposalStatus, + type ProposalDetails, +} from "@/features/governance/types"; +import { DaoIdEnum } from "@/shared/types/daos"; + +/** + * Every way the ENS ballot reaches the wallet when the gasless relayer cannot + * be used, rendered from the real VotingModal. The wallet is wagmi's mock + * connector and both the chain and Gateful are answered by a fetch stub, so + * each story runs the same code path the page does with the relayer in that + * state. + */ + +const VOTER: Address = "0x76A6D08b82034b397E7e09dAe4377C18F132BbB8"; +const MOCK_RPC_URL = "https://mock-rpc.anticapture.local"; +const VOTING_POWER = "125000000000000000000000"; + +const governorAbi = parseAbi([ + "function name() view returns (string)", + "function hasVoted(uint256 proposalId, address account) view returns (bool)", +]); + +// Mainnet's id with an RPC the stub answers, so neither the mock connector +// nor the public client ever leaves the page. +const mockMainnet = { + ...mainnet, + rpcUrls: { default: { http: [MOCK_RPC_URL] } }, +} as const; + +const AVAILABLE: RelayerStatusResponse = { + available: true, + reasons: [], + checkedAt: "2026-10-24T18:00:00.000Z", +}; + +const UNAVAILABLE: RelayerStatusResponse = { + available: false, + reasons: ["LOW_BALANCE"], + checks: { + paused: false, + rpc: "ok", + balance: "low", + rateLimiter: "ok", + pendingTransactions: "ok", + recentRelays: "ok", + }, + checkedAt: "2026-10-24T18:00:00.000Z", +}; + +const RATE_LIMIT: RelayerRateLimitResponse = { + address: VOTER, + vote: { used: 0, remaining: 3, limit: 3 }, + delegation: { used: 0, remaining: 3, limit: 3 }, + resetsAt: "2026-11-01T00:00:00.000Z", +}; + +type RelayAnswer = "refused-low-balance" | "no-answer"; + +interface Scenario { + status: RelayerStatusResponse; + relay?: RelayAnswer; +} + +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + +const answerRpc = (method: string, params: unknown[] | undefined): unknown => { + switch (method) { + case "eth_chainId": + return "0x1"; + case "eth_accounts": + case "eth_requestAccounts": + return [VOTER]; + case "eth_blockNumber": + return "0x1"; + case "eth_signTypedData_v4": + return `0x${"11".repeat(32)}${"22".repeat(32)}1b`; + case "eth_call": { + const [call] = (params ?? []) as [{ data: `0x${string}` }]; + const { functionName } = decodeFunctionData({ + abi: governorAbi, + data: call.data, + }); + return functionName === "name" + ? encodeFunctionResult({ + abi: governorAbi, + functionName: "name", + result: "ENS Governor", + }) + : encodeFunctionResult({ + abi: governorAbi, + functionName: "hasVoted", + result: false, + }); + } + default: + return null; + } +}; + +const answerGateful = (path: string, scenario: Scenario): Response => { + if (path.endsWith("/relay/status")) return json(scenario.status); + if (path.endsWith("/relay/config")) { + return json({ + minVotingPower: "1000000000000000000", + limits: { vote: 3, delegation: 3 }, + }); + } + if (path.includes("/relay/rate-limit/")) return json(RATE_LIMIT); + if (path.includes("/voting-powers/")) { + return json({ + accountId: VOTER, + votingPower: VOTING_POWER, + votesCount: 12, + proposalsCount: 0, + delegationsCount: 40, + }); + } + if (path.endsWith("/relay/vote")) { + return scenario.relay === "refused-low-balance" + ? json( + { + error: "Relayer wallet balance is too low to submit transactions", + code: "RELAYER_LOW_BALANCE", + }, + 503, + ) + : new Response("502 Bad Gateway", { status: 502 }); + } + return json({ error: "not mocked" }, 404); +}; + +/** Routes every request the modal makes to this story's scenario. */ +const stubNetwork = (scenario: Scenario) => { + const realFetch = window.fetch; + window.fetch = async (input, init) => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.href + : input.url; + + if (url.startsWith(MOCK_RPC_URL)) { + const body = JSON.parse(String(init?.body)) as { + id: number; + method: string; + params?: unknown[]; + }; + return json({ + jsonrpc: "2.0", + id: body.id, + result: answerRpc(body.method, body.params), + }); + } + + const gateful = url.indexOf("/api/gateful/"); + if (gateful !== -1) { + const path = new URL(url, window.location.origin).pathname; + return answerGateful(path, scenario); + } + + return realFetch(input, init); + }; + return () => { + window.fetch = realFetch; + }; +}; + +let proposalCount = 0; + +const buildProposal = (): ProposalDetails => ({ + // A proposal of its own per story: the vote store lives for the page, and + // stories share one page. + id: `${51915723803355532053585313225201979004629663092454892570706465301184496475700n + BigInt(proposalCount++)}`, + daoId: "ENS", + txHash: `0x${"ab".repeat(32)}`, + proposerAccountId: "0x983110309620D911731Ac0932219af06091b6744", + title: "[EP 6.32] Activate ENSv2", + description: "", + startBlock: 21_000_000, + endBlock: 21_045_000, + timestamp: 1_792_500_000, + status: ProposalStatus.ONGOING, + forVotes: "1400000000000000000000000", + againstVotes: "210000000000000000000000", + abstainVotes: "35000000000000000000000", + quorum: "1000000000000000000000000", + startTimestamp: 1_792_500_000, + endTimestamp: 1_793_100_000, + queuedTimestamp: null, + executedTimestamp: null, + queuedTxHash: null, + executedTxHash: null, + proposalType: null, + calldatas: [], + values: [], + targets: [], +}); + +const VotingModalStory = () => { + const [config] = useState(() => + createConfig({ + chains: [mockMainnet], + connectors: [ + mock({ + accounts: [VOTER], + features: { defaultConnected: true, reconnect: true }, + }), + ], + transports: { [mockMainnet.id]: http(MOCK_RPC_URL) }, + // One eth_call per read, so the stub answers the governor directly + // instead of decoding Multicall3 batches. + batch: { multicall: false }, + }), + ); + const [queryClient] = useState( + () => new QueryClient({ defaultOptions: { queries: { retry: false } } }), + ); + const [proposal] = useState(buildProposal); + + return ( + + +
+ {}} + proposal={proposal} + votingPower="125K" + rawVotingPower={VOTING_POWER} + decimals={18} + daoId={DaoIdEnum.ENS} + /> +
+
+
+ ); +}; + +const meta: Meta = { + title: "Governance/Gasless Vote Fallback", + parameters: { layout: "fullscreen" }, + render: () => , +}; + +export default meta; +type Story = StoryObj; + +const page = (canvasElement: HTMLElement) => + within(canvasElement.ownerDocument.body); + +const chooseFor = async (canvasElement: HTMLElement) => { + const screen = page(canvasElement); + await userEvent.click(await screen.findByText("For")); +}; + +/** 1. Relayer healthy: the vote is free, and the wallet is one click away. */ +export const GaslessAvailable: Story = { + beforeEach: () => stubNetwork({ status: AVAILABLE }), + play: async ({ canvasElement }) => { + await chooseFor(canvasElement); + const screen = page(canvasElement); + await waitFor(() => + expect( + screen.getByRole("button", { name: "Submit gas-free" }), + ).toBeEnabled(), + ); + await expect( + screen.getByText("Pay with my wallet instead"), + ).toBeInTheDocument(); + }, +}; + +/** 2. The voter opts out of the relayer and pays gas themselves. */ +export const WalletChosen: Story = { + beforeEach: () => stubNetwork({ status: AVAILABLE }), + play: async ({ canvasElement }) => { + await chooseFor(canvasElement); + const screen = page(canvasElement); + await userEvent.click( + await screen.findByText("Pay with my wallet instead"), + ); + await waitFor(() => + expect( + screen.getByRole("button", { name: "Vote with wallet" }), + ).toBeEnabled(), + ); + }, +}; + +/** 3. The status endpoint says the relayer cannot be used right now. */ +export const RelayerUnavailable: Story = { + beforeEach: () => stubNetwork({ status: UNAVAILABLE }), + play: async ({ canvasElement }) => { + await chooseFor(canvasElement); + const screen = page(canvasElement); + await screen.findByText(/Gas-free voting is unavailable right now/); + await waitFor(() => + expect( + screen.getByRole("button", { name: "Vote with wallet" }), + ).toBeEnabled(), + ); + }, +}; + +/** 4. The status said yes, but the relay was refused before broadcasting. */ +export const GaslessRefused: Story = { + beforeEach: () => + stubNetwork({ status: AVAILABLE, relay: "refused-low-balance" }), + play: async ({ canvasElement }) => { + await chooseFor(canvasElement); + const screen = page(canvasElement); + const submit = await screen.findByRole("button", { + name: "Submit gas-free", + }); + await waitFor(() => expect(submit).toBeEnabled()); + await userEvent.click(submit); + await screen.findByText(/The relayer is out of funds/); + await waitFor(() => + expect( + screen.getByRole("button", { name: "Vote with wallet" }), + ).toBeEnabled(), + ); + }, +}; + +/** 5. The relayer never answered: the vote may or may not have been sent. */ +export const RelayerNoAnswer: Story = { + beforeEach: () => stubNetwork({ status: AVAILABLE, relay: "no-answer" }), + play: async ({ canvasElement }) => { + await chooseFor(canvasElement); + const screen = page(canvasElement); + const submit = await screen.findByRole("button", { + name: "Submit gas-free", + }); + await waitFor(() => expect(submit).toBeEnabled()); + await userEvent.click(submit); + await screen.findByRole("button", { name: /Vote with my wallet instead/ }); + }, +}; + +/** 6. From there, the chain shows no vote, so the wallet takes the ballot. */ +export const FallbackFromNoAnswer: Story = { + beforeEach: () => stubNetwork({ status: AVAILABLE, relay: "no-answer" }), + play: async ({ canvasElement }) => { + await chooseFor(canvasElement); + const screen = page(canvasElement); + const submit = await screen.findByRole("button", { + name: "Submit gas-free", + }); + await waitFor(() => expect(submit).toBeEnabled()); + await userEvent.click(submit); + await userEvent.click( + await screen.findByRole("button", { + name: /Vote with my wallet instead/, + }), + ); + // The ballot's own alert, not the pending screen's, which opens with the + // same sentence. + await screen.findByText( + /did not confirm your vote\. You can still vote with your wallet/, + ); + await waitFor(() => + expect( + screen.getByRole("button", { name: "Vote with wallet" }), + ).toBeEnabled(), + ); + }, +}; diff --git a/apps/gateful/src/proxy/relayer.test.ts b/apps/gateful/src/proxy/relayer.test.ts index dda669feef..a452273d47 100644 --- a/apps/gateful/src/proxy/relayer.test.ts +++ b/apps/gateful/src/proxy/relayer.test.ts @@ -1,10 +1,15 @@ import { OpenAPIHono } from "@hono/zod-openapi"; import { vi } from "vitest"; +import { logger } from "../logger.js"; import { CircuitBreakerRegistry } from "../shared/circuit-breaker-registry"; import { relayerProxy } from "./relayer"; +vi.mock("../logger.js", () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn() }, +})); + describe("relayer proxy route", () => { const daoRelayers = new Map([ ["uni", "http://relayer-uni"], @@ -157,3 +162,247 @@ describe("relayer proxy route", () => { expect(registry.get("relayer:uni").state).toBe("OPEN"); }); }); + +describe("relayer status route", () => { + const NOW = Date.parse("2026-10-24T12:00:00.000Z"); + const daoRelayers = new Map([["uni", "http://relayer-uni"]]); + + const UNREACHABLE = { + available: false, + reasons: ["RELAYER_UNREACHABLE"], + checkedAt: "2026-10-24T12:00:00.000Z", + }; + + const RELAYER_STATUS = { + available: true, + reasons: [], + checks: { + paused: false, + rpc: "ok", + balance: "ok", + rateLimiter: "ok", + pendingTransactions: "ok", + recentRelays: "ok", + }, + balanceWei: "1000000000000000000", + thresholdWei: "100000000000000000", + checkedAt: "2026-10-24T11:59:55.000Z", + }; + + const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + + const requestedUrls = (spy: { mock: { calls: unknown[][] } }) => + spy.mock.calls.map(([arg]) => + arg instanceof Request ? arg.url : String(arg), + ); + + let app: InstanceType; + let registry: CircuitBreakerRegistry; + + beforeEach(() => { + // Only Date: the breaker's cooldown reads it, and the probe's abort + // timeout must keep running on real timers. + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(NOW); + app = new OpenAPIHono(); + registry = new CircuitBreakerRegistry(); + relayerProxy(app, daoRelayers, registry); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + /** Opens relayer:uni the way real traffic does: five failed relays in a row. */ + async function openRelayerCircuit() { + for (let i = 0; i < 5; i++) { + await app.request("/uni/relay/vote", { method: "POST" }); + } + expect(registry.get("relayer:uni").state).toBe("OPEN"); + } + + it("answers RELAYER_NOT_CONFIGURED, without fetching, for a DAO with no relayer", async () => { + const fetchSpy = vi.spyOn(global, "fetch"); + + const res = await app.request("/unknown/relay/status"); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + available: false, + reasons: ["RELAYER_NOT_CONFIGURED"], + checkedAt: "2026-10-24T12:00:00.000Z", + }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("passes the relayer's answer through verbatim, without the caller's bearer", async () => { + const fetchSpy = vi + .spyOn(global, "fetch") + .mockImplementation(async () => json(RELAYER_STATUS)); + + const res = await app.request("/UNI/relay/status", { + headers: { authorization: "Bearer tenant-token" }, + }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual(RELAYER_STATUS); + expect(requestedUrls(fetchSpy)).toEqual([ + "http://relayer-uni/relay/status", + ]); + const forwarded = fetchSpy.mock.calls[0]?.[0]; + expect( + forwarded instanceof Request && forwarded.headers.has("authorization"), + ).toBe(false); + }); + + it("passes an unavailable answer from the relayer through as well", async () => { + const lowBalance = { + ...RELAYER_STATUS, + available: false, + reasons: ["LOW_BALANCE"], + checks: { ...RELAYER_STATUS.checks, balance: "low" }, + balanceWei: "1", + }; + vi.spyOn(global, "fetch").mockImplementation(async () => json(lowBalance)); + + const res = await app.request("/uni/relay/status"); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual(lowBalance); + }); + + it.each([ + [ + "a 500", + () => json({ error: "Internal server error", code: "INTERNAL" }, 500), + ], + [ + "a structured 503", + () => + json( + { error: "Gasless relaying is paused", code: "RELAYER_PAUSED" }, + 503, + ), + ], + [ + "a non-JSON body", + () => + new Response("Bad Gateway", { + status: 200, + headers: { "content-type": "text/html" }, + }), + ], + ["a body without a boolean available", () => json({ available: "yes" })], + ["a JSON body that is not an object", () => json(true)], + ])("answers RELAYER_UNREACHABLE on %s", async (_name, respond) => { + vi.spyOn(global, "fetch").mockImplementation(async () => respond()); + + const res = await app.request("/uni/relay/status"); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual(UNREACHABLE); + }); + + it.each([ + ["a network error", () => new TypeError("fetch failed")], + [ + "a timeout", + () => + new DOMException( + "The operation was aborted due to timeout", + "TimeoutError", + ), + ], + ])("answers RELAYER_UNREACHABLE on %s", async (_name, makeError) => { + vi.spyOn(global, "fetch").mockImplementation(async () => { + throw makeError(); + }); + + const res = await app.request("/uni/relay/status"); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual(UNREACHABLE); + }); + + it("answers RELAYER_UNREACHABLE, without probing, while the relayer circuit cools down", async () => { + const fetchSpy = vi + .spyOn(global, "fetch") + .mockImplementation(async () => json({ error: "internal" }, 500)); + await openRelayerCircuit(); + + const res = await app.request("/uni/relay/status"); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual(UNREACHABLE); + expect(fetchSpy).toHaveBeenCalledTimes(5); + }); + + // A status probe must never decide whether real relay traffic is allowed: + // dashboards poll it far more often than anyone relays. + it("never moves the relayer circuit, however often the probe fails", async () => { + const fetchSpy = vi.spyOn(global, "fetch").mockImplementation(async () => { + throw new TypeError("fetch failed"); + }); + + for (let i = 0; i < 6; i++) { + const res = await app.request("/uni/relay/status"); + expect(await res.json()).toEqual(UNREACHABLE); + } + + expect(fetchSpy).toHaveBeenCalledTimes(6); + expect(registry.get("relayer:uni").state).toBe("CLOSED"); + }); + + it("probes a circuit past its cooldown without spending its half-open slot", async () => { + const fetchSpy = vi + .spyOn(global, "fetch") + .mockImplementation(async () => json({ error: "internal" }, 500)); + await openRelayerCircuit(); + vi.setSystemTime(NOW + 30_000); + fetchSpy.mockImplementation(async () => json(RELAYER_STATUS)); + + const res = await app.request("/uni/relay/status"); + + expect(await res.json()).toEqual(RELAYER_STATUS); + expect(registry.get("relayer:uni").state).toBe("OPEN"); + }); + // Dashboards poll this on every page: one outage must be one warning and + // one recovery line, not a line per request. + it("logs a relayer outage once when it starts and once when it ends", async () => { + vi.mocked(logger.warn).mockClear(); + vi.mocked(logger.info).mockClear(); + const fetchSpy = vi + .spyOn(global, "fetch") + .mockRejectedValue(new Error("connect ECONNREFUSED relayer-uni:3002")); + + await app.request("/uni/relay/status"); + await app.request("/uni/relay/status"); + await app.request("/uni/relay/status"); + + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith( + { dao: "uni", cause: "Error" }, + expect.stringContaining("unreachable"), + ); + + fetchSpy.mockResolvedValue( + new Response(JSON.stringify(RELAYER_STATUS), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + await app.request("/uni/relay/status"); + await app.request("/uni/relay/status"); + + expect(logger.info).toHaveBeenCalledTimes(1); + expect(logger.info).toHaveBeenCalledWith( + { dao: "uni" }, + "relayer status reachable again", + ); + }); +}); diff --git a/apps/gateful/src/proxy/relayer.ts b/apps/gateful/src/proxy/relayer.ts index efa9bb9cac..ef484aaebe 100644 --- a/apps/gateful/src/proxy/relayer.ts +++ b/apps/gateful/src/proxy/relayer.ts @@ -2,11 +2,57 @@ import type { OpenAPIHono } from "@hono/zod-openapi"; import type { Context } from "hono"; import { proxy as honoProxy } from "hono/proxy"; +import { logger } from "../logger.js"; import type { CircuitBreakerRegistry } from "../shared/circuit-breaker-registry.js"; import { stripAuthorization } from "./strip-authorization.js"; const PROXY_TIMEOUT_MS = 30000; +/** + * The relayer bounds each of its own checks at 3s and caches the answer, so + * anything slower than this is a relayer that is not answering. + */ +const STATUS_PROBE_TIMEOUT_MS = 5000; + +/** Reasons Gateful reports on the relayer's behalf (RelayerStatusReason). */ +type GatewayStatusReason = "RELAYER_UNREACHABLE" | "RELAYER_NOT_CONFIGURED"; + +/** + * The relayer's RelayerStatusResponse as Gateful answers it when the relayer + * cannot: unavailable, with no per-check detail since none of them ran. + */ +function unavailableStatus(reason: GatewayStatusReason) { + return { + available: false, + reasons: [reason], + checkedAt: new Date().toISOString(), + }; +} + +/** + * The relayer's status body, when it is one worth passing on: a 2xx JSON + * object whose `available` is a boolean. Anything else means the relayer did + * not really answer, so the caller reports it as unreachable instead. + */ +async function readRelayerStatus( + res: Response, +): Promise | null> { + if (!res.ok) return null; + try { + const body: unknown = await res.json(); + if ( + typeof body === "object" && + body !== null && + typeof (body as { available?: unknown }).available === "boolean" + ) { + return body as Record; + } + return null; + } catch { + return null; + } +} + /** * The relayer's global error handler wraps unhandled exceptions as * `500 { code: "INTERNAL" }`. That is a crash wearing the error contract, @@ -48,10 +94,86 @@ export function relayerProxy( daoRelayers: Map, registry: CircuitBreakerRegistry, ) { + // Ahead of /relay/* so the status probe never takes the proxy path. Its + // OpenAPI entry comes from the relayer's own spec, merged under /{dao}. + app.get("/:dao{[^/]+}/relay/status", statusHandler); app.all("/:dao{[^/]+}/relay/*", handler); app.all("/:dao{[^/]+}/config", handler); app.all("/:dao{[^/]+}/rate-limit/*", handler); + // DAOs whose relayer this Gateful is currently answering for. A dead + // relayer is invisible otherwise: voters fall back to their wallets and + // nothing errors. Logging only the transitions keeps every open dashboard's + // poll from turning one outage into a log line per request. + const answeringFor = new Set(); + + const noteUnreachable = (dao: string, cause: string) => { + if (answeringFor.has(dao)) return; + answeringFor.add(dao); + logger.warn( + { dao, cause }, + "relayer status unreachable; answering available: false on its behalf", + ); + }; + + const noteReachable = (dao: string) => { + if (answeringFor.delete(dao)) { + logger.info({ dao }, "relayer status reachable again"); + } + }; + + /** + * Always answers 200 with a RelayerStatusResponse, so the dashboard gets + * an answer to "can gasless be offered?" even when the relayer is down, + * missing or behind an open circuit, and falls back to a wallet + * transaction on anything but `available: true`. + */ + async function statusHandler(c: Context) { + const dao = c.req.param("dao")?.toLowerCase(); + const relayerUrl = dao ? daoRelayers.get(dao) : undefined; + if (!dao || !relayerUrl) { + return c.json(unavailableStatus("RELAYER_NOT_CONFIGURED"), 200); + } + + // Read-only, the way /health reads it: while the circuit is open and + // cooling down, real relay traffic is refused, so as far as a user is + // concerned the relayer is unreachable. + const breaker = registry.summary(`relayer:${dao}`); + if (breaker.state === "OPEN" && breaker.nextRetryIn > 0) { + noteUnreachable(dao, "circuit open"); + return c.json(unavailableStatus("RELAYER_UNREACHABLE"), 200); + } + + // Straight to the relayer, never through breaker.execute(): dashboards + // poll this, and a failing probe must not trip the circuit real relay + // traffic depends on or take its single HALF_OPEN slot (the same reason + // probeTarget in src/health/route.ts stays off the breaker). + let cause: string; + try { + const res = await honoProxy( + new URL("/relay/status", relayerUrl).toString(), + { + ...c.req, + // Strip the per-tenant bearer so it is never leaked to relayers. + headers: stripAuthorization(c.req.raw.headers), + signal: AbortSignal.timeout(STATUS_PROBE_TIMEOUT_MS), + }, + ); + const status = await readRelayerStatus(res); + if (status) { + noteReachable(dao); + return c.json(status, 200); + } + cause = `no valid status (HTTP ${res.status})`; + } catch (err) { + // Timeout or network error: the relayer did not answer. Only the error + // name is logged; the message can carry the relayer's internal URL. + cause = err instanceof Error ? err.name : "request failed"; + } + noteUnreachable(dao, cause); + return c.json(unavailableStatus("RELAYER_UNREACHABLE"), 200); + } + async function handler(c: Context) { const paramDao = c.req.param("dao"); if (!paramDao) { diff --git a/apps/relayer/.env.example b/apps/relayer/.env.example index fe18ef15a5..e9269ae6e3 100644 --- a/apps/relayer/.env.example +++ b/apps/relayer/.env.example @@ -23,6 +23,11 @@ MIN_VOTING_POWER=1000000000000000000000 MAX_VOTES_PER_ADDRESS_PER_MONTH=3 MAX_DELEGATIONS_PER_ADDRESS_PER_MONTH=3 +# Kill switch — only the literal "true" pauses. Refuses every relay (vote, +# delegate, queue, execute, webhook enactment) and reports PAUSED on +# GET /relay/status, so the dashboard falls back to a wallet transaction. +# GASLESS_PAUSED=true + # Server PORT=3002 diff --git a/apps/relayer/e2e/helpers/app.ts b/apps/relayer/e2e/helpers/app.ts index b88de24d77..8092f505d1 100644 --- a/apps/relayer/e2e/helpers/app.ts +++ b/apps/relayer/e2e/helpers/app.ts @@ -57,7 +57,11 @@ export async function createTestApp( ); const rateLimiter = new RateLimiter( - { incrementIfAllowed: async () => true, getCount: async () => 0 }, + { + incrementIfAllowed: async () => true, + getCount: async () => 0, + ping: async () => {}, + }, { daoName: "test", governorAddress: GOVERNOR_ADDRESS, diff --git a/apps/relayer/src/controllers/relay-delegate.ts b/apps/relayer/src/controllers/relay-delegate.ts index 3387508741..d70bc7fe24 100644 --- a/apps/relayer/src/controllers/relay-delegate.ts +++ b/apps/relayer/src/controllers/relay-delegate.ts @@ -51,7 +51,7 @@ export function relayDelegate(app: Hono, relayService: RelayService) { }, }, 503: { - description: "Relayer unavailable (low balance)", + description: "Relayer unavailable (paused or low balance)", content: { "application/json": { schema: ErrorResponseSchema }, }, diff --git a/apps/relayer/src/controllers/relay-proposal.ts b/apps/relayer/src/controllers/relay-proposal.ts index a9d733a7af..6c80d79192 100644 --- a/apps/relayer/src/controllers/relay-proposal.ts +++ b/apps/relayer/src/controllers/relay-proposal.ts @@ -26,7 +26,7 @@ const errorResponses = { content: { "application/json": { schema: ErrorResponseSchema } }, }, 503: { - description: "Relayer unavailable (low balance)", + description: "Relayer unavailable (paused or low balance)", content: { "application/json": { schema: ErrorResponseSchema } }, }, } as const; diff --git a/apps/relayer/src/controllers/relay-vote.ts b/apps/relayer/src/controllers/relay-vote.ts index af24595950..584f0042e6 100644 --- a/apps/relayer/src/controllers/relay-vote.ts +++ b/apps/relayer/src/controllers/relay-vote.ts @@ -51,7 +51,7 @@ export function relayVote(app: Hono, relayService: RelayService) { }, }, 503: { - description: "Relayer unavailable (low balance)", + description: "Relayer unavailable (paused or low balance)", content: { "application/json": { schema: ErrorResponseSchema }, }, diff --git a/apps/relayer/src/controllers/relay-webhook.test.ts b/apps/relayer/src/controllers/relay-webhook.test.ts index c0ea6b6350..5f3c1831d8 100644 --- a/apps/relayer/src/controllers/relay-webhook.test.ts +++ b/apps/relayer/src/controllers/relay-webhook.test.ts @@ -183,6 +183,18 @@ describe("POST /internal/webhook", () => { expect(outcomes).toEqual(["failed"]); }); + // The kill switch is an operator's choice, not an outage: events that + // arrive while it is on must not raise a failure each. + it("reports skipped while gasless relaying is paused", async () => { + const { app, outcomes } = createApp(async () => { + throw Errors.RELAYER_PAUSED(); + }); + const res = await post(app, webhookBody({ proposalId: "42" })); + expect(res.status).toBe(202); + await settle(); + expect(outcomes).toEqual(["skipped"]); + }); + it("returns 202 before enact settles", async () => { let resolveEnact: (value: EnactOutcome) => void; const enacted = new Promise((resolve) => { diff --git a/apps/relayer/src/controllers/relay-webhook.ts b/apps/relayer/src/controllers/relay-webhook.ts index 0329311271..b5cfdc3d7c 100644 --- a/apps/relayer/src/controllers/relay-webhook.ts +++ b/apps/relayer/src/controllers/relay-webhook.ts @@ -13,13 +13,17 @@ export type WebhookOutcome = | "failed"; // Only these RelayError codes mean "nothing to do right now" (not ready, -// would revert): expected in normal operation, not failures. Everything -// else — including TRANSACTION_REVERTED (gas was spent on a mined revert), -// other RelayErrors (404/503), and plain errors — is a failure. +// would revert, deliberately paused): expected in normal operation, not +// failures. RELAYER_PAUSED is an operator's choice, refused before anything +// is read or broadcast, so it must not raise a failure for every event that +// arrives while the kill switch is on. Everything else — including +// TRANSACTION_REVERTED (gas was spent on a mined revert), other RelayErrors +// (404/503), and plain errors — is a failure. const SKIPPED_CODES = new Set([ "INVALID_PROPOSAL_STATE", "TIMELOCK_NOT_READY", "SIMULATION_FAILED", + "RELAYER_PAUSED", ]); export interface RelayWebhookOptions { diff --git a/apps/relayer/src/controllers/status.test.ts b/apps/relayer/src/controllers/status.test.ts new file mode 100644 index 0000000000..c5dc794573 --- /dev/null +++ b/apps/relayer/src/controllers/status.test.ts @@ -0,0 +1,140 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { OpenAPIHono as Hono } from "@hono/zod-openapi"; +import { getAddress, parseEther } from "viem"; + +import { createLogger } from "@anticapture/observability"; + +import { RelayerStatusResponseSchema } from "@/schemas/status"; +import type { ChainProbe } from "@/services/chain/chain-probe"; +import { + RelayerStatusService, + type RelayerStatusDeps, +} from "@/services/status/relayer-status"; + +import { status } from "./status"; + +const silentLogger = createLogger("status-controller-test"); +silentLogger.level = "silent"; + +const NOW = Date.parse("2026-10-24T12:00:00.000Z"); +const THRESHOLD_WEI = parseEther("0.1"); + +const fail = async (): Promise => { + throw new Error("unreachable"); +}; + +const healthyChain: ChainProbe = { + chainId: async () => 1, + latestBlockTimestamp: async () => BigInt(NOW / 1000), + balance: async () => parseEther("1"), + nonce: async () => 3, +}; + +function createApp(overrides: Partial = {}) { + const app = new Hono(); + status( + app, + new RelayerStatusService( + { + paused: false, + chainId: 1, + chain: healthyChain, + relayerAddress: getAddress( + "0x1111111111111111111111111111111111111111", + ), + thresholdWei: THRESHOLD_WEI, + rateLimitStorage: { ping: async () => {} }, + relayOutcomes: { isFailing: () => false }, + ...overrides, + }, + silentLogger, + ), + ); + return app; +} + +beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(NOW); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("GET /relay/status", () => { + it("answers 200 with the status contract when the relayer can be used", async () => { + const res = await createApp().request("/relay/status"); + + expect(res.status).toBe(200); + expect(RelayerStatusResponseSchema.parse(await res.json())).toEqual({ + available: true, + reasons: [], + checks: { + paused: false, + rpc: "ok", + balance: "ok", + rateLimiter: "ok", + pendingTransactions: "ok", + recentRelays: "ok", + }, + balanceWei: parseEther("1").toString(), + thresholdWei: THRESHOLD_WEI.toString(), + checkedAt: "2026-10-24T12:00:00.000Z", + }); + }); + + it("still answers 200 when every check fails", async () => { + const app = createApp({ + paused: true, + chain: { + chainId: fail, + latestBlockTimestamp: fail, + balance: fail, + nonce: fail, + }, + rateLimitStorage: { ping: fail }, + relayOutcomes: { isFailing: () => true }, + }); + + const res = await app.request("/relay/status"); + + expect(res.status).toBe(200); + expect(RelayerStatusResponseSchema.parse(await res.json())).toEqual({ + available: false, + reasons: [ + "PAUSED", + "RPC_UNAVAILABLE", + "BALANCE_UNKNOWN", + "RATE_LIMITER_UNAVAILABLE", + "RECENT_RELAY_FAILURES", + ], + checks: { + paused: true, + rpc: "unavailable", + balance: "unknown", + rateLimiter: "unavailable", + pendingTransactions: "unknown", + recentRelays: "failing", + }, + thresholdWei: THRESHOLD_WEI.toString(), + checkedAt: "2026-10-24T12:00:00.000Z", + }); + }); + + // The dashboard and the generated SDK are written against these names. + it("publishes the route and its schemas under the contract names", () => { + const doc = createApp().getOpenAPIDocument({ + openapi: "3.0.0", + info: { title: "Anticapture Relayer", version: "1.0.0" }, + }); + + expect(Object.keys(doc.components?.schemas ?? {}).sort()).toEqual([ + "RelayerStatusReason", + "RelayerStatusResponse", + ]); + expect(doc.paths["/relay/status"]?.get?.operationId).toBe( + "getRelayerStatus", + ); + }); +}); diff --git a/apps/relayer/src/controllers/status.ts b/apps/relayer/src/controllers/status.ts new file mode 100644 index 0000000000..24ef403b97 --- /dev/null +++ b/apps/relayer/src/controllers/status.ts @@ -0,0 +1,33 @@ +import { OpenAPIHono as Hono, createRoute } from "@hono/zod-openapi"; + +import { RelayerStatusResponseSchema } from "@/schemas/status"; +import type { RelayerStatusService } from "@/services/status/relayer-status"; + +export function status( + app: Hono, + service: Pick, +) { + app.openapi( + createRoute({ + method: "get", + operationId: "getRelayerStatus", + path: "/relay/status", + summary: "Gasless relayer availability", + description: + "Answers whether the gasless relayer can be used right now, from every check the relayer can run on itself: " + + "the pause switch, the RPC, the wallet balance, the rate limiter, pending transactions and recent broadcast failures. " + + "Always answers 200 and reports a failed check in the body, never as an error status, so polling it cannot trip " + + "Gateful's circuit breaker for real relay traffic. Results are reused for a few seconds.", + tags: ["system"], + responses: { + 200: { + description: "Relayer availability and the outcome of each check", + content: { + "application/json": { schema: RelayerStatusResponseSchema }, + }, + }, + }, + }), + async (c) => c.json(await service.getStatus(), 200), + ); +} diff --git a/apps/relayer/src/env.ts b/apps/relayer/src/env.ts index 76ec3f8d59..e8711d91d0 100644 --- a/apps/relayer/src/env.ts +++ b/apps/relayer/src/env.ts @@ -2,6 +2,8 @@ import dotenv from "dotenv"; import { getAddress, isAddress, isHex } from "viem"; import { z } from "zod"; +import { envFlag } from "@/schemas/flag"; + dotenv.config(); const envSchema = z.object({ @@ -37,6 +39,13 @@ const envSchema = z.object({ .regex(/^\d+$/, "Must be a non-negative decimal") .default("100000000000000000"), + // Operator kill switch: refuses every gasless relay and reports PAUSED on + // GET /relay/status, so the dashboard sends users to their own wallet (for + // instance while a leaked key is rotated). true/1/yes/on pause; a value + // envFlag does not recognise stops the relayer at boot, which also takes + // gasless offline, rather than leaving the switch silently off. + GASLESS_PAUSED: envFlag, + // Per address, per calendar month (UTC). Optional; each defaults to DEFAULT_RELAY_LIMIT (3). // .positive() rejects 0/negatives: a 0 limit would block all relays — omit the var to use the default instead. MAX_VOTES_PER_ADDRESS_PER_MONTH: z.coerce diff --git a/apps/relayer/src/errors.ts b/apps/relayer/src/errors.ts index acadcf48f5..9636fff1d5 100644 --- a/apps/relayer/src/errors.ts +++ b/apps/relayer/src/errors.ts @@ -29,6 +29,8 @@ export class RelayError extends Error { // send". A new code added here is treated as ambiguous over there until it is // added to that set, so a pre-broadcast code shipped with a 5xx status will // strand users on "transaction may have been sent" until both sides agree. +// RELAYER_PAUSED is the earliest of them: the GASLESS_PAUSED kill switch +// refuses before a signature is even recovered or the chain is read. export const Errors = { INSUFFICIENT_VOTING_POWER: (min: string) => new RelayError( @@ -48,6 +50,8 @@ export const Errors = { "RELAYER_LOW_BALANCE", 503, ), + RELAYER_PAUSED: () => + new RelayError("Gasless relaying is paused", "RELAYER_PAUSED", 503), INVALID_CONTRACT: () => new RelayError( "Target contract is not in the whitelist", diff --git a/apps/relayer/src/index.ts b/apps/relayer/src/index.ts index 9111442aa6..45a14f45cd 100644 --- a/apps/relayer/src/index.ts +++ b/apps/relayer/src/index.ts @@ -20,6 +20,7 @@ import { relayDelegate } from "@/controllers/relay-delegate"; import { relayProposal } from "@/controllers/relay-proposal"; import { relayVote } from "@/controllers/relay-vote"; import { relayWebhook } from "@/controllers/relay-webhook"; +import { status } from "@/controllers/status"; import { env } from "@/env"; import { RelayError } from "@/errors"; import { createClient } from "redis"; @@ -29,11 +30,18 @@ import { resolveRelayLimits, } from "@/services/guards/rate-limiter"; import { ChainStateService } from "@/services/chain/chain-state"; +import { ViemChainProbe } from "@/services/chain/chain-probe"; import { ViemGovernorGateway } from "@/services/chain/governor-gateway"; import { ProposalEnactmentService } from "@/services/proposals/proposal-enactment"; import { AnticaptureProposalSource } from "@/services/proposals/proposal-source"; import { RelayService } from "@/services/relay"; import { SignatureVerifier } from "@/services/guards/signature-verifier"; +import { + RelayOutcomeTracker, + trackBroadcasts, +} from "@/services/status/relay-outcome-tracker"; +import { RelayerStatusService } from "@/services/status/relayer-status"; +import { registerStatusMetrics } from "@/services/status/status-metrics"; import { createLocalSigner } from "@/signer/local-signer"; import { exporter, meterProvider } from "@/instrumentation"; @@ -58,7 +66,13 @@ async function main() { transport: http(env.RPC_URL), }); - const signer = createLocalSigner(env.RELAYER_PRIVATE_KEY, chain, env.RPC_URL); + // Every broadcast with the relayer wallet goes through this signer, so + // GET /relay/status can tell when they keep failing. + const relayOutcomes = new RelayOutcomeTracker(); + const signer = trackBroadcasts( + createLocalSigner(env.RELAYER_PRIVATE_KEY, chain, env.RPC_URL), + relayOutcomes, + ); const relayerAddress = await signer.getAddress(); // --- Services --- @@ -118,6 +132,7 @@ async function main() { BigInt(env.MIN_VOTING_POWER), governorAddress, tokenAddress, + env.GASLESS_PAUSED, ), ); @@ -179,7 +194,10 @@ async function main() { env.ANTICAPTURE_API_KEY, ), ), - { minBalanceWei: BigInt(env.MIN_RELAYER_BALANCE_WEI) }, + { + minBalanceWei: BigInt(env.MIN_RELAYER_BALANCE_WEI), + paused: env.GASLESS_PAUSED, + }, ), ); @@ -207,6 +225,23 @@ async function main() { relayerAddress, thresholdWei: BigInt(env.MIN_RELAYER_BALANCE_WEI), }); + const relayerStatus = wrapWithTracing( + new RelayerStatusService({ + paused: env.GASLESS_PAUSED, + chainId: env.CHAIN_ID, + chain: wrapWithTracing(new ViemChainProbe(publicClient)), + relayerAddress, + thresholdWei: BigInt(env.MIN_RELAYER_BALANCE_WEI), + rateLimitStorage, + relayOutcomes, + }), + ); + status(app, relayerStatus); + registerStatusMetrics( + meterProvider.getMeter("anticapture-relayer"), + relayerStatus, + env.DAO_NAME, + ); // --- OpenAPI docs --- app.doc("/docs", { @@ -224,6 +259,7 @@ async function main() { token: tokenAddress, voteLimitPerMonth: relayLimits.vote, delegationLimitPerMonth: relayLimits.delegation, + gaslessPaused: env.GASLESS_PAUSED, }, "Relayer starting", ); diff --git a/apps/relayer/src/repository/rate-limit-storage.test.ts b/apps/relayer/src/repository/rate-limit-storage.test.ts index 0ce7fc3c17..6cd7c65812 100644 --- a/apps/relayer/src/repository/rate-limit-storage.test.ts +++ b/apps/relayer/src/repository/rate-limit-storage.test.ts @@ -17,6 +17,12 @@ const MAX_PER_MONTH = 3; */ class FakeRedis implements RedisClient { private counters = new Map(); + down = false; + + async ping(): Promise { + if (this.down) throw new Error("connection refused"); + return "PONG"; + } async incr(key: string): Promise { const next = (this.counters.get(key) ?? 0) + 1; @@ -207,6 +213,18 @@ describe("RedisRateLimitStorage.getCount", () => { }); }); +describe("RedisRateLimitStorage.ping", () => { + it("resolves when Redis answers", async () => { + await expect(store.ping()).resolves.toBeUndefined(); + }); + + it("rejects when Redis does not answer", async () => { + redis.down = true; + + await expect(store.ping()).rejects.toThrow("connection refused"); + }); +}); + describe("secondsUntilNextUtcMonth", () => { it("counts the seconds remaining from mid-month to the next month start", () => { // 2026-06-15T00:00:00Z → next boundary is 2026-07-01T00:00:00Z = 16 days later. diff --git a/apps/relayer/src/repository/rate-limit-storage.ts b/apps/relayer/src/repository/rate-limit-storage.ts index 038ba086dd..e648b7d65c 100644 --- a/apps/relayer/src/repository/rate-limit-storage.ts +++ b/apps/relayer/src/repository/rate-limit-storage.ts @@ -21,12 +21,15 @@ export interface GetCountParams { export interface RateLimitStorage { incrementIfAllowed(params: IncrementIfAllowedParams): Promise; getCount(params: GetCountParams): Promise; + /** Resolves once the store answers a round trip; rejects when it cannot. */ + ping(): Promise; } export interface RedisClient { incr(key: string): Promise; expire(key: string, seconds: number): Promise; get(key: string): Promise; + ping(): Promise; } /** @@ -119,4 +122,13 @@ export class RedisRateLimitStorage implements RateLimitStorage { const raw = await this.redis.get(monthKey); return raw === null ? 0 : Number(raw); } + + /** + * A PING round trip, the cheapest proof that Redis answers. While the client + * is reconnecting it queues commands rather than failing them, so callers + * must bound this with their own timeout. + */ + async ping(): Promise { + await this.redis.ping(); + } } diff --git a/apps/relayer/src/schemas/flag.test.ts b/apps/relayer/src/schemas/flag.test.ts new file mode 100644 index 0000000000..d7f77990ec --- /dev/null +++ b/apps/relayer/src/schemas/flag.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; + +import { envFlag } from "@/schemas/flag"; + +describe("envFlag", () => { + it.each(["true", "TRUE", " True ", "1", "yes", "YES", "on", "On"])( + "reads %j as on", + (value) => { + expect(envFlag.parse(value)).toBe(true); + }, + ); + + it.each([undefined, "", " ", "false", "FALSE", "0", "no", "off"])( + "reads %j as off", + (value) => { + expect(envFlag.parse(value)).toBe(false); + }, + ); + + // A typo must stop the relayer at boot, not leave the kill switch off. + it.each(["ture", "paused", "2", "enabled"])("refuses %j", (value) => { + expect(envFlag.safeParse(value).success).toBe(false); + }); +}); diff --git a/apps/relayer/src/schemas/flag.ts b/apps/relayer/src/schemas/flag.ts new file mode 100644 index 0000000000..f09cbff895 --- /dev/null +++ b/apps/relayer/src/schemas/flag.ts @@ -0,0 +1,27 @@ +import { z } from "zod"; + +const ON = new Set(["true", "1", "yes", "on"]); +const OFF = new Set(["false", "0", "no", "off", ""]); + +/** + * An on/off environment flag that refuses anything it does not recognise. + * + * A kill switch read as `value === "true"` is silently off for "TRUE", "1" or + * "yes", which is exactly what an operator types under pressure. Reading the + * common spellings, and failing the parse on any other value, makes a typo + * stop the process at boot instead of leaving the switch off. An unset or + * empty variable is off. + */ +export const envFlag = z + .string() + .optional() + .transform((value, ctx) => { + const normalized = (value ?? "").trim().toLowerCase(); + if (ON.has(normalized)) return true; + if (OFF.has(normalized)) return false; + ctx.addIssue({ + code: "custom", + message: `Expected true/false, 1/0, yes/no or on/off, got "${value}"`, + }); + return z.NEVER; + }); diff --git a/apps/relayer/src/schemas/status.ts b/apps/relayer/src/schemas/status.ts new file mode 100644 index 0000000000..f36c9684ca --- /dev/null +++ b/apps/relayer/src/schemas/status.ts @@ -0,0 +1,95 @@ +import { z } from "@hono/zod-openapi"; + +export const RelayerStatusReasonSchema = z + .enum([ + "PAUSED", + "RPC_UNAVAILABLE", + "WRONG_CHAIN", + "CHAIN_STALE", + "LOW_BALANCE", + "BALANCE_UNKNOWN", + "RATE_LIMITER_UNAVAILABLE", + "PENDING_TRANSACTIONS_STUCK", + "RECENT_RELAY_FAILURES", + "RELAYER_UNREACHABLE", + "RELAYER_NOT_CONFIGURED", + ]) + .openapi("RelayerStatusReason", { + description: + "Why gasless relaying cannot be used right now. " + + "PAUSED: an operator switched gasless relaying off. " + + "RPC_UNAVAILABLE: the relayer's RPC did not answer. " + + "WRONG_CHAIN: the RPC serves a different chain than the one the relayer signs for. " + + "CHAIN_STALE: the RPC's latest block is more than two minutes old. " + + "LOW_BALANCE: the relayer wallet holds less than the configured threshold. " + + "BALANCE_UNKNOWN: the relayer wallet's balance could not be read. " + + "RATE_LIMITER_UNAVAILABLE: the rate limiter's storage did not answer, so every relay would be refused. " + + "PENDING_TRANSACTIONS_STUCK: a transaction from the relayer wallet has sat unmined for five minutes. " + + "RECENT_RELAY_FAILURES: three or more broadcasts failed within ten minutes, with no success since. " + + "RELAYER_UNREACHABLE and RELAYER_NOT_CONFIGURED are only emitted by Gateful on the relayer's behalf: " + + "the relayer did not answer (or its circuit is open), or the DAO has no relayer configured.", + }); + +const RelayerStatusChecksSchema = z.object({ + paused: z.boolean().openapi({ + description: + "True when an operator has switched gasless relaying off (GASLESS_PAUSED).", + }), + rpc: z.enum(["ok", "unavailable", "wrong_chain", "stale"]).openapi({ + description: + "Whether the relayer's RPC answers, serves the chain the relayer signs for, and has a latest block no older than two minutes.", + }), + balance: z.enum(["ok", "low", "unknown"]).openapi({ + description: + "Whether the relayer wallet's native balance is at or above the configured threshold; unknown when it could not be read.", + }), + rateLimiter: z.enum(["ok", "unavailable"]).openapi({ + description: + "Whether the rate limiter's storage answered a round trip. Every relay goes through it, so without it every relay is refused.", + }), + pendingTransactions: z.enum(["ok", "stuck", "unknown"]).openapi({ + description: + "Whether the relayer wallet's pending nonce has sat ahead of its mined nonce, with the mined one not moving, for five minutes. " + + "Unknown when the nonces could not be read, which on its own does not make the relayer unavailable.", + }), + recentRelays: z.enum(["ok", "failing"]).openapi({ + description: + "Whether three or more broadcasts failed within ten minutes with no success since. " + + "Clears ten minutes after the latest failure, or on the next successful broadcast.", + }), +}); + +export const RelayerStatusResponseSchema = z + .object({ + available: z.boolean().openapi({ + description: + "True only when every check passed, which is also when reasons is empty. " + + "Anything other than an answer with available: true means gasless relaying should not be offered.", + }), + reasons: z.array(RelayerStatusReasonSchema).openapi({ + description: "Every check that failed. Empty when available.", + }), + checks: RelayerStatusChecksSchema.optional().openapi({ + description: + "Outcome of each check the relayer ran. Absent when Gateful answered on the relayer's behalf.", + }), + balanceWei: z.string().optional().openapi({ + format: "bigint", + description: + "Current native balance of the relayer wallet, as a decimal string (uint256, in wei). Absent when it could not be read.", + }), + thresholdWei: z.string().optional().openapi({ + format: "bigint", + description: + "Minimum native balance the relayer must hold to be considered funded, as a decimal string (uint256, in wei). Absent when Gateful answered on the relayer's behalf.", + }), + checkedAt: z.iso.datetime().openapi({ + description: + "ISO 8601 timestamp of when the checks ran. Answers are reused for a few seconds, so it can trail the request.", + }), + }) + .openapi("RelayerStatusResponse"); + +export type RelayerStatusReason = z.infer; +export type RelayerStatusChecks = z.infer; +export type RelayerStatusResponse = z.infer; diff --git a/apps/relayer/src/services/chain/chain-probe.ts b/apps/relayer/src/services/chain/chain-probe.ts new file mode 100644 index 0000000000..eca631aa8c --- /dev/null +++ b/apps/relayer/src/services/chain/chain-probe.ts @@ -0,0 +1,48 @@ +import type { Address, PublicActions } from "viem"; + +/** + * Domain-level view of the chain reads the status check makes, so the + * status service (and its tests) never deal with viem generics. + */ +export interface ChainProbe { + /** Chain id the RPC reports through eth_chainId. */ + chainId(): Promise; + /** Timestamp of the latest block, in unix seconds. */ + latestBlockTimestamp(): Promise; + balance(address: Address): Promise; + /** + * Transactions sent from the address: mined ones only at "latest", plus + * the ones the node still holds in its mempool at "pending". + */ + nonce(address: Address, blockTag: "latest" | "pending"): Promise; +} + +/** The viem actions the probe needs. A real PublicClient satisfies this. */ +export type ChainProbeReader = Pick< + PublicActions, + "getChainId" | "getBlock" | "getBalance" | "getTransactionCount" +>; + +export class ViemChainProbe implements ChainProbe { + constructor(private client: ChainProbeReader) {} + + async chainId(): Promise { + return this.client.getChainId(); + } + + async latestBlockTimestamp(): Promise { + const { timestamp } = await this.client.getBlock(); + return timestamp; + } + + async balance(address: Address): Promise { + return this.client.getBalance({ address }); + } + + async nonce( + address: Address, + blockTag: "latest" | "pending", + ): Promise { + return this.client.getTransactionCount({ address, blockTag }); + } +} diff --git a/apps/relayer/src/services/guards/rate-limiter.test.ts b/apps/relayer/src/services/guards/rate-limiter.test.ts index 23c11400a8..668247f53c 100644 --- a/apps/relayer/src/services/guards/rate-limiter.test.ts +++ b/apps/relayer/src/services/guards/rate-limiter.test.ts @@ -32,6 +32,7 @@ function makeStore(): RateLimitStorage { async getCount({ address, operation }) { return counters.get(`${address}:${operation}`) ?? 0; }, + async ping() {}, }; } @@ -105,6 +106,7 @@ describe("RateLimiter", () => { const brokenStore: RateLimitStorage = { incrementIfAllowed: () => Promise.reject(new Error("connection refused")), getCount: () => Promise.reject(new Error("connection refused")), + ping: () => Promise.reject(new Error("connection refused")), }; const brokenLimiter = new RateLimiter(brokenStore, { diff --git a/apps/relayer/src/services/proposals/proposal-enactment.test.ts b/apps/relayer/src/services/proposals/proposal-enactment.test.ts index 5212aa21b2..46c9fefe1b 100644 --- a/apps/relayer/src/services/proposals/proposal-enactment.test.ts +++ b/apps/relayer/src/services/proposals/proposal-enactment.test.ts @@ -18,6 +18,7 @@ import type { ReceiptOutcome, } from "@/services/chain/governor-gateway"; import { SimulationRevertError } from "@/services/chain/governor-gateway"; +import { trackBroadcasts } from "@/services/status/relay-outcome-tracker"; import { RelayerSigner } from "@/signer/types"; import { ProposalEnactmentService } from "./proposal-enactment"; @@ -89,6 +90,7 @@ function createService( overrides: { governor?: Partial; source?: ProposalSource; + paused?: boolean; } = {}, ) { const { governor, simulated } = createStubGovernor(overrides.governor); @@ -97,7 +99,7 @@ function createService( governor, signer, overrides.source ?? createStubSource().source, - { minBalanceWei: parseEther("0.1").valueOf() }, + { minBalanceWei: parseEther("0.1").valueOf(), paused: overrides.paused }, silentLogger, ); return { service, sent, simulated }; @@ -638,3 +640,92 @@ describe("ProposalEnactmentService.enact", () => { await expectRelayError(service.enact("999"), "PROPOSAL_NOT_FOUND", 404); }); }); + +describe("ProposalEnactmentService kill switch", () => { + it.each(["queue", "execute", "enact"] as const)( + "refuses %s with 503 RELAYER_PAUSED before reading the chain or the API", + async (method) => { + const stateReads: bigint[] = []; + const { source, fetched } = createStubSource(); + const { service, sent } = createService({ + paused: true, + governor: { + state: async (proposalId) => { + stateReads.push(proposalId); + return ProposalState.Succeeded; + }, + }, + source, + }); + + await expectRelayError( + service[method](PROPOSAL_ID.toString()), + "RELAYER_PAUSED", + 503, + ); + expect(stateReads).toEqual([]); + expect(fetched).toEqual([]); + expect(sent).toEqual([]); + }, + ); +}); + +// index.ts wraps the signer with trackBroadcasts, which is how enactment +// feeds GET /relay/status. Only an actual broadcast may count either way. +describe("ProposalEnactmentService broadcast tracking", () => { + function createTrackedService( + sendTransaction: RelayerSigner["sendTransaction"], + governor: Partial = {}, + ) { + const outcomes: ("success" | "failure")[] = []; + const signer = trackBroadcasts( + { getAddress: async () => RELAYER, sendTransaction }, + { + recordSuccess: () => outcomes.push("success"), + recordFailure: () => outcomes.push("failure"), + }, + ); + const service = new ProposalEnactmentService( + createStubGovernor(governor).governor, + signer, + createStubSource().source, + { minBalanceWei: parseEther("0.1").valueOf() }, + silentLogger, + ); + return { service, outcomes }; + } + + it("records a success for a broadcast queue()", async () => { + const { service, outcomes } = createTrackedService(async () => TX_HASH); + + await service.queue(PROPOSAL_ID.toString()); + + expect(outcomes).toEqual(["success"]); + }); + + it("records a failure for a broadcast answered as RELAYER_LOW_BALANCE", async () => { + const { service, outcomes } = createTrackedService(async () => { + throw new InsufficientFundsError(); + }); + + await expectRelayError( + service.queue(PROPOSAL_ID.toString()), + "RELAYER_LOW_BALANCE", + 503, + ); + expect(outcomes).toEqual(["failure"]); + }); + + it("records nothing when the balance guard refuses before broadcasting", async () => { + const { service, outcomes } = createTrackedService(async () => TX_HASH, { + ethBalance: async () => parseEther("0.01"), + }); + + await expectRelayError( + service.queue(PROPOSAL_ID.toString()), + "RELAYER_LOW_BALANCE", + 503, + ); + expect(outcomes).toEqual([]); + }); +}); diff --git a/apps/relayer/src/services/proposals/proposal-enactment.ts b/apps/relayer/src/services/proposals/proposal-enactment.ts index bbab9570d6..f633db9e28 100644 --- a/apps/relayer/src/services/proposals/proposal-enactment.ts +++ b/apps/relayer/src/services/proposals/proposal-enactment.ts @@ -24,6 +24,8 @@ import type { ProposalSource } from "./proposal-source"; export interface ProposalEnactmentConfig { /** Below this relayer balance the service refuses to broadcast. */ minBalanceWei: bigint; + /** The GASLESS_PAUSED kill switch: every enactment is refused while set. */ + paused?: boolean; } type EnactmentAction = "queue" | "execute"; @@ -65,10 +67,12 @@ export class ProposalEnactmentService { ) {} async queue(proposalId: string): Promise<{ txHash: Hash }> { + this.assertNotPaused(); return this.dispatch("queue", proposalId); } async execute(proposalId: string): Promise<{ txHash: Hash }> { + this.assertNotPaused(); return this.dispatch("execute", proposalId); } @@ -81,6 +85,7 @@ export class ProposalEnactmentService { * the two reads still ends in a 409 rather than a wasted broadcast. */ async enact(proposalId: string): Promise { + this.assertNotPaused(); const state = await this.governor.state(BigInt(proposalId)); if (state === null) { throw Errors.PROPOSAL_NOT_FOUND(proposalId); @@ -94,6 +99,16 @@ export class ProposalEnactmentService { return { action: "skipped", state: ProposalState[state] }; } + /** + * Runs first in every entry point, so a paused relayer neither reads the + * chain nor calls the Anticapture API, let alone broadcasts. + */ + private assertNotPaused(): void { + if (this.config.paused) { + throw Errors.RELAYER_PAUSED(); + } + } + private dispatch( action: EnactmentAction, proposalId: string, diff --git a/apps/relayer/src/services/relay.test.ts b/apps/relayer/src/services/relay.test.ts index ce2277f02d..ecba667971 100644 --- a/apps/relayer/src/services/relay.test.ts +++ b/apps/relayer/src/services/relay.test.ts @@ -4,6 +4,7 @@ import { ContractFunctionRevertedError, getAddress, HttpRequestError, + InsufficientFundsError, type Hash, type Hex, } from "viem"; @@ -12,6 +13,7 @@ import { governorAbi } from "@/abi/governor"; import { erc20VotesAbi } from "@/abi/token"; import { RelayError } from "@/errors"; import { RelayService } from "./relay"; +import { trackBroadcasts } from "./status/relay-outcome-tracker"; import type { RelayerSigner } from "@/signer/types"; import type { ISignatureVerifier } from "./guards/signature-verifier"; import type { IChainStateService } from "./chain/chain-state"; @@ -112,6 +114,7 @@ function createService( rateLimiter?: IRateLimiter; minVotingPower?: bigint; chainReader?: ChainReader; + paused?: boolean; } = {}, ): RelayService { return new RelayService( @@ -123,9 +126,36 @@ function createService( overrides.minVotingPower ?? 0n, GOVERNOR, TOKEN, + overrides.paused, ); } +/** Signature verifier that records whether any signer was recovered. */ +function createRecordingVerifier() { + const recovered: string[] = []; + const verifier = createStubSignatureVerifier({ + recoverVoteSigner: async () => { + recovered.push("vote"); + return VOTER; + }, + recoverDelegationSigner: async () => { + recovered.push("delegation"); + return VOTER; + }, + }); + return { verifier, recovered }; +} + +/** Outcome recorder in the shape trackBroadcasts feeds. */ +function createRecorder() { + const outcomes: ("success" | "failure")[] = []; + return { + outcomes, + recordSuccess: () => outcomes.push("success"), + recordFailure: () => outcomes.push("failure"), + }; +} + describe("RelayService", () => { describe("relayVote", () => { const voteParams = { @@ -217,6 +247,20 @@ describe("RelayService", () => { await expect(service.relayVote(voteParams)).rejects.toBe(transportError); }); + + it("answers 503 RELAYER_PAUSED, before recovering the voter, while paused", async () => { + const { verifier, recovered } = createRecordingVerifier(); + const service = createService({ + paused: true, + signatureVerifier: verifier, + }); + + const err = await caught(service.relayVote(voteParams)); + + expect(err).toBeInstanceOf(RelayError); + expect(err).toMatchObject({ code: "RELAYER_PAUSED", status: 503 }); + expect(recovered).toEqual([]); + }); }); describe("relayDelegation", () => { @@ -306,5 +350,75 @@ describe("RelayService", () => { ); expect(sent).toBe(false); }); + + it("answers 503 RELAYER_PAUSED, before recovering the delegator, while paused", async () => { + const { verifier, recovered } = createRecordingVerifier(); + const service = createService({ + paused: true, + signatureVerifier: verifier, + }); + + const err = await caught(service.relayDelegation(delegateParams)); + + expect(err).toBeInstanceOf(RelayError); + expect(err).toMatchObject({ code: "RELAYER_PAUSED", status: 503 }); + expect(recovered).toEqual([]); + }); + }); + + // index.ts wraps the signer with trackBroadcasts, which is how relays feed + // GET /relay/status. Only an actual broadcast may count either way. + describe("broadcast outcomes", () => { + const voteParams = { + proposalId: 1n, + support: 1, + v: 27, + r: SAMPLE_R, + s: SAMPLE_S, + }; + + it("records a success for a relayed vote", async () => { + const recorder = createRecorder(); + const service = createService({ + signer: trackBroadcasts(createStubSigner(), recorder), + }); + + await service.relayVote(voteParams); + + expect(recorder.outcomes).toEqual(["success"]); + }); + + it("records a failure for a broadcast answered as RELAYER_LOW_BALANCE", async () => { + const recorder = createRecorder(); + const service = createService({ + signer: trackBroadcasts( + createStubSigner({ + sendTransaction: async () => { + throw new InsufficientFundsError(); + }, + }), + recorder, + ), + }); + + const err = await caught(service.relayVote(voteParams)); + + expect(err).toMatchObject({ code: "RELAYER_LOW_BALANCE", status: 503 }); + expect(recorder.outcomes).toEqual(["failure"]); + }); + + it("records nothing for a relay refused before broadcasting", async () => { + const recorder = createRecorder(); + const service = createService({ + signer: trackBroadcasts(createStubSigner(), recorder), + chainState: createStubChainState({ getVotingPower: async () => 0n }), + minVotingPower: 1000n, + }); + + await expect(service.relayVote(voteParams)).rejects.toThrow( + "minimum voting power", + ); + expect(recorder.outcomes).toEqual([]); + }); }); }); diff --git a/apps/relayer/src/services/relay.ts b/apps/relayer/src/services/relay.ts index bba189c25a..de5c5b6ae8 100644 --- a/apps/relayer/src/services/relay.ts +++ b/apps/relayer/src/services/relay.ts @@ -66,9 +66,14 @@ export class RelayService { private minVotingPower: bigint, private governorAddress: Address, private tokenAddress: Address, + /** The GASLESS_PAUSED kill switch: every relay is refused while set. */ + private paused = false, ) {} async relayVote(params: RelayVoteParams): Promise { + // 0. Kill switch — before anything is recovered, read or spent + if (this.paused) throw Errors.RELAYER_PAUSED(); + // 1. Recover voter from signature const voter = await this.signatureVerifier.recoverVoteSigner(params); @@ -111,6 +116,9 @@ export class RelayService { } async relayDelegation(params: RelayDelegateParams): Promise { + // 0. Kill switch — before anything is recovered, read or spent + if (this.paused) throw Errors.RELAYER_PAUSED(); + // 1. Recover delegator from signature const delegator = await this.signatureVerifier.recoverDelegationSigner(params); diff --git a/apps/relayer/src/services/status/relay-outcome-tracker.test.ts b/apps/relayer/src/services/status/relay-outcome-tracker.test.ts new file mode 100644 index 0000000000..07c5e2e047 --- /dev/null +++ b/apps/relayer/src/services/status/relay-outcome-tracker.test.ts @@ -0,0 +1,142 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { getAddress, InsufficientFundsError, type Hash } from "viem"; + +import type { RelayerSigner } from "@/signer/types"; + +import { + RECENT_FAILURE_THRESHOLD, + RECENT_FAILURE_WINDOW_MS, + RelayOutcomeTracker, + trackBroadcasts, +} from "./relay-outcome-tracker"; + +const NOW = Date.parse("2026-10-24T12:00:00.000Z"); +const MINUTE = 60_000; +const RELAYER = getAddress("0x1111111111111111111111111111111111111111"); +const TX_HASH: Hash = `0x${"ab".repeat(32)}`; +const TX = { + to: getAddress("0x2222222222222222222222222222222222222222"), + data: "0xdeadbeef" as const, +}; + +beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("RelayOutcomeTracker", () => { + let tracker: RelayOutcomeTracker; + + beforeEach(() => { + tracker = new RelayOutcomeTracker(); + }); + + const failAt = (offsetMs: number) => { + vi.setSystemTime(NOW + offsetMs); + tracker.recordFailure(); + }; + + const isFailingAt = (offsetMs: number) => { + vi.setSystemTime(NOW + offsetMs); + return tracker.isFailing(); + }; + + it("is not failing before any broadcast", () => { + expect(tracker.isFailing()).toBe(false); + }); + + it("is not failing below the threshold", () => { + for (let i = 0; i < RECENT_FAILURE_THRESHOLD - 1; i++) failAt(i * MINUTE); + + expect(isFailingAt(RECENT_FAILURE_THRESHOLD * MINUTE)).toBe(false); + }); + + it("is failing once the threshold is reached inside the window", () => { + failAt(0); + failAt(1 * MINUTE); + failAt(2 * MINUTE); + + expect(isFailingAt(2 * MINUTE)).toBe(true); + }); + + it("does not add up failures further apart than the window", () => { + failAt(0); + failAt(6 * MINUTE); + failAt(RECENT_FAILURE_WINDOW_MS + 1 * MINUTE); + + expect(isFailingAt(RECENT_FAILURE_WINDOW_MS + 1 * MINUTE)).toBe(false); + }); + + it("stays failing until a window has passed since the latest failure", () => { + failAt(0); + failAt(1 * MINUTE); + failAt(2 * MINUTE); + + expect(isFailingAt(2 * MINUTE + RECENT_FAILURE_WINDOW_MS - 1)).toBe(true); + expect(isFailingAt(2 * MINUTE + RECENT_FAILURE_WINDOW_MS)).toBe(false); + }); + + it("clears on the next success, and counts afresh after it", () => { + failAt(0); + failAt(1 * MINUTE); + failAt(2 * MINUTE); + + tracker.recordSuccess(); + expect(tracker.isFailing()).toBe(false); + + failAt(3 * MINUTE); + failAt(4 * MINUTE); + expect(tracker.isFailing()).toBe(false); + }); +}); + +describe("trackBroadcasts", () => { + function createRecorder() { + const outcomes: ("success" | "failure")[] = []; + return { + outcomes, + recordSuccess: () => outcomes.push("success"), + recordFailure: () => outcomes.push("failure"), + }; + } + + it("passes the transaction through and records a success", async () => { + const sent: unknown[] = []; + const signer: RelayerSigner = { + getAddress: async () => RELAYER, + sendTransaction: async (tx) => { + sent.push(tx); + return TX_HASH; + }, + }; + const recorder = createRecorder(); + const tracked = trackBroadcasts(signer, recorder); + + const hash = await tracked.sendTransaction(TX); + + expect(hash).toBe(TX_HASH); + expect(sent).toEqual([TX]); + expect(recorder.outcomes).toEqual(["success"]); + expect(await tracked.getAddress()).toBe(RELAYER); + }); + + it("records a failure and rethrows it untouched", async () => { + const insufficientFunds = new InsufficientFundsError(); + const signer: RelayerSigner = { + getAddress: async () => RELAYER, + sendTransaction: async () => { + throw insufficientFunds; + }, + }; + const recorder = createRecorder(); + + await expect( + trackBroadcasts(signer, recorder).sendTransaction(TX), + ).rejects.toBe(insufficientFunds); + expect(recorder.outcomes).toEqual(["failure"]); + }); +}); diff --git a/apps/relayer/src/services/status/relay-outcome-tracker.ts b/apps/relayer/src/services/status/relay-outcome-tracker.ts new file mode 100644 index 0000000000..ab64428de1 --- /dev/null +++ b/apps/relayer/src/services/status/relay-outcome-tracker.ts @@ -0,0 +1,75 @@ +import type { RelayerSigner } from "@/signer/types"; + +/** Broadcast failures, inside one window, that mark recent relays as failing. */ +export const RECENT_FAILURE_THRESHOLD = 3; + +/** + * How close together those failures must land, and how long the failing + * state outlives the latest of them when nothing succeeds in between. + */ +export const RECENT_FAILURE_WINDOW_MS = 10 * 60_000; + +/** + * Remembers whether the relayer wallet's recent broadcasts keep failing, + * which no single probe of the chain can show: a node that rejects every + * transaction for a nonce or fee reason, a key that no longer controls the + * funds. Only broadcasts count. A request refused before it reached the + * signer (bad signature, rate limit, simulation revert, voting power) says + * nothing about the wallet, so it is neither a failure nor a success here. + * + * Kept in memory on purpose: a restart, which is how a rotated key or a + * fixed RPC reaches the relayer, should start from a clean slate. + */ +export class RelayOutcomeTracker { + // Failures since the last success, pruned to the window that ends at the + // latest one, so a slow trickle of unrelated failures never adds up. + private failures: number[] = []; + + recordSuccess(): void { + this.failures = []; + } + + recordFailure(): void { + const now = Date.now(); + this.failures = [ + ...this.failures.filter((at) => now - at < RECENT_FAILURE_WINDOW_MS), + now, + ]; + } + + isFailing(): boolean { + const latest = this.failures.at(-1); + return ( + latest !== undefined && + Date.now() - latest < RECENT_FAILURE_WINDOW_MS && + this.failures.length >= RECENT_FAILURE_THRESHOLD + ); + } +} + +/** + * Wraps the relayer signer so every broadcast reports its outcome. Votes, + * delegations and proposal enactment all send through this one signer, so + * none of them has to remember to report, and a failure is recorded whatever + * it becomes upstream: an InsufficientFundsError still counts here even + * though the services answer it as 503 RELAYER_LOW_BALANCE. + */ +export function trackBroadcasts( + signer: RelayerSigner, + tracker: Pick, +): RelayerSigner { + return { + getAddress: () => signer.getAddress(), + + async sendTransaction(tx) { + try { + const hash = await signer.sendTransaction(tx); + tracker.recordSuccess(); + return hash; + } catch (err) { + tracker.recordFailure(); + throw err; + } + }, + }; +} diff --git a/apps/relayer/src/services/status/relayer-status.test.ts b/apps/relayer/src/services/status/relayer-status.test.ts new file mode 100644 index 0000000000..2b88a358ad --- /dev/null +++ b/apps/relayer/src/services/status/relayer-status.test.ts @@ -0,0 +1,397 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { getAddress, parseEther } from "viem"; + +import { createLogger } from "@anticapture/observability"; + +import type { + RelayerStatusChecks, + RelayerStatusReason, + RelayerStatusResponse, +} from "@/schemas/status"; +import type { ChainProbe } from "@/services/chain/chain-probe"; + +import { + MAX_BLOCK_AGE_MS, + RelayerStatusService, + STATUS_CACHE_TTL_MS, + STATUS_CHECK_TIMEOUT_MS, + STUCK_PENDING_AFTER_MS, + type RelayerStatusDeps, +} from "./relayer-status"; + +const silentLogger = createLogger("relayer-status-test"); +silentLogger.level = "silent"; + +const RELAYER = getAddress("0x1111111111111111111111111111111111111111"); +const THRESHOLD_WEI = parseEther("0.1"); +const BALANCE_WEI = parseEther("1"); +const NOW = Date.parse("2026-10-24T12:00:00.000Z"); +const MINUTE = 60_000; + +const HEALTHY_CHECKS: RelayerStatusChecks = { + paused: false, + rpc: "ok", + balance: "ok", + rateLimiter: "ok", + pendingTransactions: "ok", + recentRelays: "ok", +}; + +const fail = async (): Promise => { + throw new Error("unreachable"); +}; +const hang = (): Promise => new Promise(() => {}); + +/** A healthy mainnet RPC whose latest block is always fresh; any read can be swapped. */ +function createChain(overrides: Partial = {}): ChainProbe { + return { + chainId: async () => 1, + latestBlockTimestamp: async () => BigInt(Math.floor(Date.now() / 1000)), + balance: async () => BALANCE_WEI, + nonce: async () => 7, + ...overrides, + }; +} + +function createService(overrides: Partial = {}) { + return new RelayerStatusService( + { + paused: false, + chainId: 1, + chain: createChain(), + relayerAddress: RELAYER, + thresholdWei: THRESHOLD_WEI, + rateLimitStorage: { ping: async () => {} }, + relayOutcomes: { isFailing: () => false }, + ...overrides, + }, + silentLogger, + ); +} + +function expectedStatus({ + reasons = [], + checks = {}, + balanceWei = BALANCE_WEI, +}: { + reasons?: RelayerStatusReason[]; + checks?: Partial; + balanceWei?: bigint | null; +} = {}): RelayerStatusResponse { + return { + available: reasons.length === 0, + reasons, + checks: { ...HEALTHY_CHECKS, ...checks }, + ...(balanceWei === null ? {} : { balanceWei: balanceWei.toString() }), + thresholdWei: THRESHOLD_WEI.toString(), + checkedAt: new Date(NOW).toISOString(), + }; +} + +beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("RelayerStatusService", () => { + it("reports available when every check passes", async () => { + const service = createService(); + + const status = await service.getStatus(); + + expect(status).toEqual(expectedStatus()); + }); + + it.each< + [ + RelayerStatusReason, + Partial, + Partial, + ] + >([ + ["PAUSED", { paused: true }, { paused: true }], + [ + "RPC_UNAVAILABLE", + { chain: createChain({ chainId: fail }) }, + { rpc: "unavailable" }, + ], + [ + "WRONG_CHAIN", + { chain: createChain({ chainId: async () => 11155111 }) }, + { rpc: "wrong_chain" }, + ], + [ + "CHAIN_STALE", + { + chain: createChain({ + latestBlockTimestamp: async () => + BigInt((NOW - MAX_BLOCK_AGE_MS - 1000) / 1000), + }), + }, + { rpc: "stale" }, + ], + [ + "RATE_LIMITER_UNAVAILABLE", + { rateLimitStorage: { ping: fail } }, + { rateLimiter: "unavailable" }, + ], + [ + "RECENT_RELAY_FAILURES", + { relayOutcomes: { isFailing: () => true } }, + { recentRelays: "failing" }, + ], + ])("reports %s", async (reason, deps, checks) => { + const service = createService(deps); + + const status = await service.getStatus(); + + expect(status).toEqual(expectedStatus({ reasons: [reason], checks })); + }); + + it("accepts a latest block exactly at the maximum age", async () => { + const service = createService({ + chain: createChain({ + latestBlockTimestamp: async () => + BigInt((NOW - MAX_BLOCK_AGE_MS) / 1000), + }), + }); + + const status = await service.getStatus(); + + expect(status).toEqual(expectedStatus()); + }); + + it("reports LOW_BALANCE with the balance it read", async () => { + const service = createService({ + chain: createChain({ balance: async () => THRESHOLD_WEI - 1n }), + }); + + const status = await service.getStatus(); + + expect(status).toEqual( + expectedStatus({ + reasons: ["LOW_BALANCE"], + checks: { balance: "low" }, + balanceWei: THRESHOLD_WEI - 1n, + }), + ); + }); + + it("accepts a balance exactly at the threshold", async () => { + const service = createService({ + chain: createChain({ balance: async () => THRESHOLD_WEI }), + }); + + const status = await service.getStatus(); + + expect(status).toEqual(expectedStatus({ balanceWei: THRESHOLD_WEI })); + }); + + it("reports BALANCE_UNKNOWN, without a balance, when it cannot be read", async () => { + const service = createService({ chain: createChain({ balance: fail }) }); + + const status = await service.getStatus(); + + expect(status).toEqual( + expectedStatus({ + reasons: ["BALANCE_UNKNOWN"], + checks: { balance: "unknown" }, + balanceWei: null, + }), + ); + }); + + it("reports a check that hangs as failed once its timeout elapses", async () => { + const service = createService({ + chain: createChain({ latestBlockTimestamp: hang }), + rateLimitStorage: { ping: hang }, + }); + + const pending = service.getStatus(); + await vi.advanceTimersByTimeAsync(STATUS_CHECK_TIMEOUT_MS); + + expect(await pending).toEqual( + expectedStatus({ + reasons: ["RPC_UNAVAILABLE", "RATE_LIMITER_UNAVAILABLE"], + checks: { rpc: "unavailable", rateLimiter: "unavailable" }, + }), + ); + }); + + it("answers, with every failed check in a fixed order, when everything fails", async () => { + const service = createService({ + paused: true, + chain: { + chainId: fail, + latestBlockTimestamp: fail, + balance: fail, + nonce: fail, + }, + rateLimitStorage: { ping: fail }, + relayOutcomes: { isFailing: () => true }, + }); + + const status = await service.getStatus(); + + // An unreadable nonce is "unknown" without a reason of its own: the RPC + // check already reports the RPC that failed it. + expect(status).toEqual( + expectedStatus({ + reasons: [ + "PAUSED", + "RPC_UNAVAILABLE", + "BALANCE_UNKNOWN", + "RATE_LIMITER_UNAVAILABLE", + "RECENT_RELAY_FAILURES", + ], + checks: { + paused: true, + rpc: "unavailable", + balance: "unknown", + rateLimiter: "unavailable", + pendingTransactions: "unknown", + recentRelays: "failing", + }, + balanceWei: null, + }), + ); + }); + + describe("caching", () => { + it("reuses an answer for the cache window, then runs the checks again", async () => { + let balanceReads = 0; + let balance = BALANCE_WEI; + const service = createService({ + chain: createChain({ + balance: async () => { + balanceReads++; + return balance; + }, + }), + }); + + const first = await service.getStatus(); + balance = THRESHOLD_WEI - 1n; + vi.setSystemTime(NOW + STATUS_CACHE_TTL_MS - 1); + const cached = await service.getStatus(); + vi.setSystemTime(NOW + STATUS_CACHE_TTL_MS); + const refreshed = await service.getStatus(); + + expect(cached).toBe(first); + expect(refreshed).toEqual({ + ...expectedStatus({ + reasons: ["LOW_BALANCE"], + checks: { balance: "low" }, + balanceWei: THRESHOLD_WEI - 1n, + }), + checkedAt: new Date(NOW + STATUS_CACHE_TTL_MS).toISOString(), + }); + expect(balanceReads).toBe(2); + }); + + it("shares the run in flight between concurrent callers", async () => { + let chainIdReads = 0; + const service = createService({ + chain: createChain({ + chainId: async () => { + chainIdReads++; + return 1; + }, + }), + }); + + const [first, second] = await Promise.all([ + service.getStatus(), + service.getStatus(), + ]); + + expect(second).toBe(first); + expect(chainIdReads).toBe(1); + }); + }); + + describe("pending transactions", () => { + function createNonceService() { + const nonces = { pending: 7, latest: 7, failing: false }; + const service = createService({ + chain: createChain({ + nonce: async (_address, blockTag) => { + if (nonces.failing) throw new Error("rpc unreachable"); + return nonces[blockTag]; + }, + }), + }); + + // Probes are at least a cache window apart, so every one runs the checks. + const probeAt = async (offsetMs: number) => { + vi.setSystemTime(NOW + offsetMs); + const { checks, reasons } = await service.getStatus(); + return { pendingTransactions: checks?.pendingTransactions, reasons }; + }; + return { nonces, probeAt }; + } + + const OK = { pendingTransactions: "ok", reasons: [] }; + const STUCK = { + pendingTransactions: "stuck", + reasons: ["PENDING_TRANSACTIONS_STUCK"], + }; + + it("reports stuck once the pending nonce has sat ahead of the mined one for five minutes", async () => { + const { nonces, probeAt } = createNonceService(); + nonces.pending = 8; + + expect(await probeAt(0)).toEqual(OK); + expect( + await probeAt(STUCK_PENDING_AFTER_MS - STATUS_CACHE_TTL_MS), + ).toEqual(OK); + expect(await probeAt(STUCK_PENDING_AFTER_MS)).toEqual(STUCK); + }); + + it("restarts the clock when the gap closes", async () => { + const { nonces, probeAt } = createNonceService(); + + nonces.pending = 8; + expect(await probeAt(0)).toEqual(OK); + nonces.pending = 7; + expect(await probeAt(3 * MINUTE)).toEqual(OK); + nonces.pending = 8; + expect(await probeAt(4 * MINUTE)).toEqual(OK); + + expect(await probeAt(STUCK_PENDING_AFTER_MS)).toEqual(OK); + expect(await probeAt(4 * MINUTE + STUCK_PENDING_AFTER_MS)).toEqual(STUCK); + }); + + it("restarts the clock while the mined nonce keeps moving", async () => { + const { nonces, probeAt } = createNonceService(); + + nonces.pending = 9; + expect(await probeAt(0)).toEqual(OK); + // Still a gap, but a transaction was mined: the queue is draining. + nonces.pending = 10; + nonces.latest = 8; + expect(await probeAt(3 * MINUTE)).toEqual(OK); + + expect(await probeAt(STUCK_PENDING_AFTER_MS)).toEqual(OK); + expect(await probeAt(3 * MINUTE + STUCK_PENDING_AFTER_MS)).toEqual(STUCK); + }); + + it("reports unknown, without a reason, when the nonces cannot be read, and keeps the clock", async () => { + const { nonces, probeAt } = createNonceService(); + + nonces.pending = 8; + expect(await probeAt(0)).toEqual(OK); + nonces.failing = true; + expect(await probeAt(3 * MINUTE)).toEqual({ + pendingTransactions: "unknown", + reasons: [], + }); + nonces.failing = false; + expect(await probeAt(STUCK_PENDING_AFTER_MS)).toEqual(STUCK); + }); + }); +}); diff --git a/apps/relayer/src/services/status/relayer-status.ts b/apps/relayer/src/services/status/relayer-status.ts new file mode 100644 index 0000000000..afbf9e9c2b --- /dev/null +++ b/apps/relayer/src/services/status/relayer-status.ts @@ -0,0 +1,240 @@ +import type { Address } from "viem"; + +import { createLogger, type Logger } from "@anticapture/observability"; + +import type { RateLimitStorage } from "@/repository/rate-limit-storage"; +import type { + RelayerStatusChecks, + RelayerStatusReason, + RelayerStatusResponse, +} from "@/schemas/status"; +import type { ChainProbe } from "@/services/chain/chain-probe"; + +import type { RelayOutcomeTracker } from "./relay-outcome-tracker"; + +/** Upper bound on each check, so one hung dependency cannot hold the answer. */ +export const STATUS_CHECK_TIMEOUT_MS = 3_000; + +/** How long an answer is reused, so polling dashboards cannot hammer the RPC. */ +export const STATUS_CACHE_TTL_MS = 10_000; + +/** + * A latest block older than this means the RPC stopped following the chain: + * mainnet produces one every 12 seconds. + */ +export const MAX_BLOCK_AGE_MS = 120_000; + +/** + * How long the wallet's pending nonce may sit ahead of its mined nonce, + * without the mined one moving, before its transactions count as stuck. + */ +export const STUCK_PENDING_AFTER_MS = 5 * 60_000; + +export interface RelayerStatusDeps { + /** The GASLESS_PAUSED kill switch. */ + paused: boolean; + /** The chain the relayer signs for (CHAIN_ID). */ + chainId: number; + chain: ChainProbe; + relayerAddress: Address; + /** Below this native balance the wallet counts as underfunded. */ + thresholdWei: bigint; + rateLimitStorage: Pick; + relayOutcomes: Pick; +} + +interface BalanceCheck { + state: RelayerStatusChecks["balance"]; + wei?: bigint; +} + +/** + * Rejects once `ms` elapse. viem retries a failing RPC for longer than that, + * and the Redis client queues commands while it reconnects, so neither would + * fail fast on its own. The abandoned call keeps running in the background. + */ +function withTimeout(promise: Promise, ms: number): Promise { + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`check timed out after ${ms}ms`)), + ms, + ); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} + +/** + * Answers "can the gasless relayer be used right now?" for every failure the + * relayer can detect in itself, so the dashboard falls back to a regular + * wallet transaction instead of letting a user sign a relay that will not + * land. Never throws: a check that fails or hangs is itself the answer. + */ +export class RelayerStatusService { + private cached: { + at: number; + status: Promise; + } | null = null; + + // When the pending nonce was first seen ahead of the mined one, and the + // mined nonce at that moment. A gap whose mined nonce moves on is a busy + // queue draining rather than a stuck one, so it restarts the clock. + private pendingGap: { since: number; latestNonce: number } | null = null; + + // Reasons from the previous run, so only changes of availability are + // logged rather than every poll of a degraded relayer. + private lastReasons = ""; + + constructor( + private deps: RelayerStatusDeps, + private logger: Logger = createLogger("relayer-status"), + ) {} + + /** + * Concurrent callers inside the cache window share one run, including the + * one still in flight, so the checks never overlap. + */ + async getStatus(): Promise { + const now = Date.now(); + if (this.cached && now - this.cached.at < STATUS_CACHE_TTL_MS) { + return this.cached.status; + } + const status = this.runChecks(now); + this.cached = { at: now, status }; + return status; + } + + private async runChecks(now: number): Promise { + const [rpc, balance, rateLimiter, pendingTransactions] = await Promise.all([ + this.checkRpc(now), + this.checkBalance(), + this.checkRateLimiter(), + this.checkPendingTransactions(now), + ]); + const checks: RelayerStatusChecks = { + paused: this.deps.paused, + rpc, + balance: balance.state, + rateLimiter, + pendingTransactions, + recentRelays: this.deps.relayOutcomes.isFailing() ? "failing" : "ok", + }; + const reasons = reasonsFor(checks); + this.logAvailabilityChange(reasons); + + return { + available: reasons.length === 0, + reasons, + checks, + ...(balance.wei !== undefined + ? { balanceWei: balance.wei.toString() } + : {}), + thresholdWei: this.deps.thresholdWei.toString(), + checkedAt: new Date(now).toISOString(), + }; + } + + private async checkRpc(now: number): Promise { + try { + const [chainId, blockTimestamp] = await withTimeout( + Promise.all([ + this.deps.chain.chainId(), + this.deps.chain.latestBlockTimestamp(), + ]), + STATUS_CHECK_TIMEOUT_MS, + ); + if (chainId !== this.deps.chainId) return "wrong_chain"; + return now - Number(blockTimestamp) * 1000 > MAX_BLOCK_AGE_MS + ? "stale" + : "ok"; + } catch { + return "unavailable"; + } + } + + private async checkBalance(): Promise { + try { + const wei = await withTimeout( + this.deps.chain.balance(this.deps.relayerAddress), + STATUS_CHECK_TIMEOUT_MS, + ); + return { state: wei >= this.deps.thresholdWei ? "ok" : "low", wei }; + } catch { + return { state: "unknown" }; + } + } + + private async checkRateLimiter(): Promise< + RelayerStatusChecks["rateLimiter"] + > { + try { + await withTimeout( + this.deps.rateLimitStorage.ping(), + STATUS_CHECK_TIMEOUT_MS, + ); + return "ok"; + } catch { + return "unavailable"; + } + } + + private async checkPendingTransactions( + now: number, + ): Promise { + let pending: number; + let latest: number; + try { + [pending, latest] = await withTimeout( + Promise.all([ + this.deps.chain.nonce(this.deps.relayerAddress, "pending"), + this.deps.chain.nonce(this.deps.relayerAddress, "latest"), + ]), + STATUS_CHECK_TIMEOUT_MS, + ); + } catch { + // A failed read is no evidence the gap closed, so the clock keeps + // running; the RPC check already reports an RPC that is down. + return "unknown"; + } + + if (pending <= latest) { + this.pendingGap = null; + return "ok"; + } + if (this.pendingGap?.latestNonce !== latest) { + this.pendingGap = { since: now, latestNonce: latest }; + } + return now - this.pendingGap.since >= STUCK_PENDING_AFTER_MS + ? "stuck" + : "ok"; + } + + private logAvailabilityChange(reasons: RelayerStatusReason[]): void { + const key = reasons.join(","); + if (key === this.lastReasons) return; + this.lastReasons = key; + if (reasons.length > 0) { + this.logger.warn({ reasons }, "gasless relaying unavailable"); + } else { + this.logger.info("gasless relaying available again"); + } + } +} + +function reasonsFor(checks: RelayerStatusChecks): RelayerStatusReason[] { + const reasons: RelayerStatusReason[] = []; + if (checks.paused) reasons.push("PAUSED"); + if (checks.rpc === "unavailable") reasons.push("RPC_UNAVAILABLE"); + if (checks.rpc === "wrong_chain") reasons.push("WRONG_CHAIN"); + if (checks.rpc === "stale") reasons.push("CHAIN_STALE"); + if (checks.balance === "low") reasons.push("LOW_BALANCE"); + if (checks.balance === "unknown") reasons.push("BALANCE_UNKNOWN"); + if (checks.rateLimiter === "unavailable") { + reasons.push("RATE_LIMITER_UNAVAILABLE"); + } + if (checks.pendingTransactions === "stuck") { + reasons.push("PENDING_TRANSACTIONS_STUCK"); + } + if (checks.recentRelays === "failing") reasons.push("RECENT_RELAY_FAILURES"); + return reasons; +} diff --git a/apps/relayer/src/services/status/status-metrics.test.ts b/apps/relayer/src/services/status/status-metrics.test.ts new file mode 100644 index 0000000000..9e2cabbc48 --- /dev/null +++ b/apps/relayer/src/services/status/status-metrics.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; + +import { + createObservabilityProvider, + PrometheusSerializer, +} from "@anticapture/observability"; + +import type { RelayerStatusResponse } from "@/schemas/status"; + +import { registerStatusMetrics } from "./status-metrics"; + +const scrape = async (status: RelayerStatusResponse): Promise => { + const { exporter, meterProvider } = createObservabilityProvider( + "status-metrics-test", + ); + registerStatusMetrics( + meterProvider.getMeter("status-metrics-test"), + { getStatus: async () => status }, + "ens", + ); + const result = await exporter.collect(); + return new PrometheusSerializer().serialize(result.resourceMetrics); +}; + +describe("registerStatusMetrics", () => { + it("reports an available relayer as 1 with no reasons", async () => { + const text = await scrape({ + available: true, + reasons: [], + checkedAt: "2026-10-24T12:00:00.000Z", + }); + + expect(text).toMatch(/relayer_gasless_available\{dao="ens"\} 1/); + expect(text).not.toMatch(/relayer_gasless_unavailable_reason\{/); + }); + + it("reports an unavailable relayer as 0 with one series per reason", async () => { + const text = await scrape({ + available: false, + reasons: ["LOW_BALANCE", "RPC_UNAVAILABLE"], + checkedAt: "2026-10-24T12:00:00.000Z", + }); + + expect(text).toMatch(/relayer_gasless_available\{dao="ens"\} 0/); + expect(text).toMatch( + /relayer_gasless_unavailable_reason\{dao="ens",reason="LOW_BALANCE"\} 1/, + ); + expect(text).toMatch( + /relayer_gasless_unavailable_reason\{dao="ens",reason="RPC_UNAVAILABLE"\} 1/, + ); + }); +}); diff --git a/apps/relayer/src/services/status/status-metrics.ts b/apps/relayer/src/services/status/status-metrics.ts new file mode 100644 index 0000000000..2cf049274a --- /dev/null +++ b/apps/relayer/src/services/status/status-metrics.ts @@ -0,0 +1,48 @@ +import type { meterProvider } from "@/instrumentation"; +import type { RelayerStatusService } from "@/services/status/relayer-status"; + +type Meter = ReturnType<(typeof meterProvider)["getMeter"]>; + +export type StatusMeter = Pick< + Meter, + "createObservableGauge" | "addBatchObservableCallback" +>; + +/** + * Publishes GET /relay/status on /metrics, so a relayer that stops being able + * to sponsor transactions pages someone instead of only changing what the + * dashboard offers. Voters keep voting from their wallets either way, which + * is exactly why nobody notices without this: the outage is silent. + * + * The gauges are read on each scrape and go through getStatus(), so they + * share its cache and its never-throw guarantee, and the status is evaluated + * on Prometheus' schedule even when no dashboard is polling. + */ +export const registerStatusMetrics = ( + meter: StatusMeter, + status: Pick, + dao: string, +): void => { + const available = meter.createObservableGauge("relayer_gasless_available", { + description: + "1 when GET /relay/status reports gasless relaying available, 0 otherwise.", + }); + const unavailableReason = meter.createObservableGauge( + "relayer_gasless_unavailable_reason", + { + description: + "1 for each reason GET /relay/status currently reports gasless relaying unavailable.", + }, + ); + + meter.addBatchObservableCallback( + async (observer) => { + const current = await status.getStatus(); + observer.observe(available, current.available ? 1 : 0, { dao }); + for (const reason of current.reasons) { + observer.observe(unavailableReason, 1, { dao, reason }); + } + }, + [available, unavailableReason], + ); +}; diff --git a/infra/monitoring/alerts.yml b/infra/monitoring/alerts.yml index 608a8913ff..11ca3871e9 100644 --- a/infra/monitoring/alerts.yml +++ b/infra/monitoring/alerts.yml @@ -342,3 +342,27 @@ groups: 2 minutes (threshold: 1%). Users are experiencing server errors. Check application logs for stack traces, database connectivity, and any recent deployments that may have introduced regressions. + + # The gasless relayer reports it cannot sponsor transactions (unfunded + # wallet, RPC or Redis down, stuck nonce, repeated broadcast failures). + # Voters keep voting from their own wallets, which is what makes this + # silent without an alert. A deliberate GASLESS_PAUSED is left out: the + # operator who set it already knows. + - alert: GaslessRelayerUnavailable + expr: > + max by (dao) (relayer_gasless_available) == 0 + unless on (dao) + max by (dao) (relayer_gasless_unavailable_reason{reason="PAUSED"}) == 1 + for: 10m + keep_firing_for: 15m + labels: + severity: warning + annotations: + summary: "[WARNING] gasless relaying is unavailable for {{ $labels.dao }}" + description: > + GET /relay/status on the {{ $labels.dao }} relayer has reported available: false + for more than 10 minutes, so the dashboard sends every voter to their own wallet. + relayer_gasless_unavailable_reason{dao="{{ $labels.dao }}"} lists why: fund the + relayer wallet for LOW_BALANCE, check eRPC for RPC_UNAVAILABLE or CHAIN_STALE, + Redis for RATE_LIMITER_UNAVAILABLE, and the relayer's pending transactions for + PENDING_TRANSACTIONS_STUCK or RECENT_RELAY_FAILURES.