Skip to content

Commit aedef5c

Browse files
Bojan131Bojan131
committed
Fix blockchain transaction error handling
- Add retry logic for transient errors (socket hang up, connection reset, 502/503, fee too low) - Only bump gas price on nonce/fee errors, not on network errors - Add gas estimation retries for transient RPC failures - Add 60s timeout on RPC provider connection to prevent node hanging on unresponsive RPCs - Fix BigNumber NaN bug in gas price calculation during retry - Fix Gnosis EIP-1559 gas params and double gwei-parsing in gas price comparison - Add RPC failover for blockchain event fetching (try all providers before failing) - Log warning when blockchain events are missed due to large block gaps Made-with: Cursor
1 parent 4e25e00 commit aedef5c

5 files changed

Lines changed: 170 additions & 28 deletions

File tree

‎src/commands/blockchain-event-listener/blockchain-event-listener-command.js‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,13 @@ class BlockchainEventListenerCommand extends Command {
6565
);
6666

6767
if (eventsMissed) {
68-
// TODO: Add some logic for missed events in the future
68+
const missedFrom = (lastCheckedBlockRecord?.lastCheckedBlock ?? 0) + 1;
69+
this.logger.warn(
70+
`[EVENT LISTENER] Blockchain events missed on ${blockchainId}! ` +
71+
`Gap too large: blocks ${missedFrom}–${currentBlock} ` +
72+
`(${currentBlock - missedFrom + 1} blocks). ` +
73+
`Publish finality for assets created during this window will not complete.`,
74+
);
6975
}
7076

7177
if (newEvents.length !== 0) {

‎src/constants/constants.js‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -715,6 +715,15 @@ export const EXPECTED_TRANSACTION_ERRORS = {
715715
NONCE_TOO_LOW: 'nonce too low',
716716
REPLACEMENT_UNDERPRICED: 'replacement transaction underpriced',
717717
ALREADY_KNOWN: 'already known',
718+
EXECUTION_FAILED: 'transaction execution fails',
719+
FEE_TOO_LOW: 'feetoolow',
720+
SOCKET_HANG_UP: 'socket hang up',
721+
ECONNRESET: 'econnreset',
722+
ECONNREFUSED: 'econnrefused',
723+
SERVER_ERROR: 'server error',
724+
BAD_GATEWAY: '502',
725+
SERVICE_UNAVAILABLE: '503',
726+
EXPECT_BLOCK_NUMBER: 'expect block number from id',
718727
};
719728

720729
/**

‎src/modules/blockchain-events/implementation/ot-ethers/ot-ethers.js‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,38 @@ class OtEthers extends BlockchainEventsService {
5656
return blockchainProviders[randomIndex];
5757
}
5858

59+
_getShuffledProviders(blockchain) {
60+
const blockchainProviders = this.providers[blockchain];
61+
if (!blockchainProviders || blockchainProviders.length === 0) {
62+
throw new Error(`No providers available for blockchain: ${blockchain}`);
63+
}
64+
const shuffled = [...blockchainProviders];
65+
for (let i = shuffled.length - 1; i > 0; i -= 1) {
66+
const j = Math.floor(Math.random() * (i + 1));
67+
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
68+
}
69+
return shuffled;
70+
}
71+
72+
async _sendWithFailover(blockchain, method, params) {
73+
const providers = this._getShuffledProviders(blockchain);
74+
let lastError;
75+
for (const provider of providers) {
76+
try {
77+
return await provider.send(method, params);
78+
} catch (error) {
79+
lastError = error;
80+
this.logger.warn(
81+
`RPC provider failed for ${method} on ${blockchain}: ${error.message}. ` +
82+
`Trying next provider (${providers.indexOf(provider) + 1}/${
83+
providers.length
84+
})...`,
85+
);
86+
}
87+
}
88+
throw lastError;
89+
}
90+
5991
async _initializeContracts() {
6092
this.contracts = {};
6193

@@ -156,8 +188,7 @@ class OtEthers extends BlockchainEventsService {
156188
const toBlockParam = ethers.BigNumber.from(toBlock)
157189
.toHexString()
158190
.replace(/^0x0+/, '0x');
159-
const provider = this._getRandomProvider(blockchain);
160-
const newLogs = await provider.send('eth_getLogs', [
191+
const newLogs = await this._sendWithFailover(blockchain, 'eth_getLogs', [
161192
{
162193
address: contractAddresses,
163194
fromBlock: fromBlockParam,

‎src/modules/blockchain/implementation/gnosis/gnosis-service.js‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,16 +44,32 @@ class GnosisService extends Web3Service {
4444
);
4545
return this.defaultGasPrice;
4646
}
47-
if (
48-
gasPrice &&
49-
ethers.utils.parseUnits(gasPrice.toString(), 'gwei').gt(this.defaultGasPrice)
50-
) {
47+
if (gasPrice && gasPrice.gt && gasPrice.gt(this.defaultGasPrice)) {
5148
return gasPrice;
5249
}
5350

5451
return this.defaultGasPrice;
5552
}
5653

54+
buildTransactionGasParams(gasPrice) {
55+
const minPriorityFee = ethers.BigNumber.from(2_000_000_000);
56+
57+
let maxPriorityFeePerGas = minPriorityFee;
58+
if (ethers.BigNumber.isBigNumber(gasPrice)) {
59+
const derived = gasPrice.div(5);
60+
if (derived.gt(minPriorityFee)) {
61+
maxPriorityFeePerGas = derived;
62+
}
63+
}
64+
65+
const maxFeePerGas =
66+
ethers.BigNumber.isBigNumber(gasPrice) && gasPrice.gt(maxPriorityFeePerGas)
67+
? gasPrice
68+
: maxPriorityFeePerGas.mul(2);
69+
70+
return { maxFeePerGas, maxPriorityFeePerGas };
71+
}
72+
5773
async healthCheck() {
5874
try {
5975
const blockNumber = await this.getBlockNumber();

‎src/modules/blockchain/implementation/web3-service.js‎

Lines changed: 101 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -522,6 +522,10 @@ class Web3Service {
522522
}
523523
}
524524

525+
buildTransactionGasParams(gasPrice) {
526+
return { gasPrice };
527+
}
528+
525529
async callContractFunction(contractInstance, functionName, args, contractName = null) {
526530
const maxNumberOfRetries = 3;
527531
const retryDelayInSec = 12;
@@ -567,11 +571,35 @@ class Web3Service {
567571
let retryCount = 0;
568572
const maxRetries = 3;
569573

570-
try {
571-
/* eslint-disable no-await-in-loop */
572-
gasLimit = await contractInstance.estimateGas[functionName](...args);
573-
} catch (error) {
574-
this._decodeEstimateGasError(contractInstance, functionName, error, args);
574+
for (let estimateAttempt = 0; estimateAttempt < 3; estimateAttempt += 1) {
575+
try {
576+
/* eslint-disable no-await-in-loop */
577+
gasLimit = await contractInstance.estimateGas[functionName](...args);
578+
break;
579+
} catch (error) {
580+
const errMsg = error.message?.toLowerCase() ?? '';
581+
const isTransient =
582+
errMsg.includes(EXPECTED_TRANSACTION_ERRORS.EXECUTION_FAILED.toLowerCase()) ||
583+
errMsg.includes(EXPECTED_TRANSACTION_ERRORS.FEE_TOO_LOW.toLowerCase()) ||
584+
errMsg.includes(EXPECTED_TRANSACTION_ERRORS.SOCKET_HANG_UP) ||
585+
errMsg.includes(EXPECTED_TRANSACTION_ERRORS.ECONNRESET) ||
586+
errMsg.includes(EXPECTED_TRANSACTION_ERRORS.ECONNREFUSED) ||
587+
errMsg.includes(EXPECTED_TRANSACTION_ERRORS.SERVER_ERROR) ||
588+
errMsg.includes(EXPECTED_TRANSACTION_ERRORS.BAD_GATEWAY) ||
589+
errMsg.includes(EXPECTED_TRANSACTION_ERRORS.SERVICE_UNAVAILABLE) ||
590+
errMsg.includes(EXPECTED_TRANSACTION_ERRORS.EXPECT_BLOCK_NUMBER);
591+
if (isTransient && estimateAttempt < 2) {
592+
this.logger.warn(
593+
`Gas estimation for ${functionName} failed with transient error on ${this.getBlockchainId()}, ` +
594+
`retrying (${estimateAttempt + 1}/3): ${error.message}`,
595+
);
596+
await new Promise((r) => {
597+
setTimeout(r, 2000);
598+
});
599+
continue;
600+
}
601+
this._decodeEstimateGasError(contractInstance, functionName, error, args);
602+
}
575603
}
576604

577605
gasLimit = gasLimit ?? ethers.utils.parseUnits('900', 'kwei');
@@ -590,12 +618,12 @@ class Web3Service {
590618
}${retryCount > 0 ? ` (retry ${retryCount})` : ''}`,
591619
);
592620

621+
const txOverrides = this.buildTransactionGasParams(gasPrice);
622+
txOverrides.gasLimit = gasLimit;
623+
593624
const tx = await contractInstance
594625
.connect(operationalWallet)
595-
[functionName](...args, {
596-
gasPrice,
597-
gasLimit,
598-
});
626+
[functionName](...args, txOverrides);
599627

600628
try {
601629
result = await this.provider.waitForTransaction(
@@ -608,38 +636,90 @@ class Web3Service {
608636
await this.provider.call(tx, tx.blockNumber);
609637
}
610638
} catch (error) {
639+
if (
640+
error.message
641+
.toLowerCase()
642+
.includes(EXPECTED_TRANSACTION_ERRORS.TIMEOUT_EXCEEDED.toLowerCase())
643+
) {
644+
const existingReceipt = await this.provider.getTransactionReceipt(tx.hash);
645+
if (existingReceipt) {
646+
this.logger.info(
647+
`Transaction ${functionName} (${tx.hash}) confirmed despite timeout. Block: ${existingReceipt.blockNumber}`,
648+
);
649+
if (existingReceipt.status === 0) {
650+
await this.provider.call(tx, existingReceipt.blockNumber);
651+
}
652+
return existingReceipt;
653+
}
654+
throw error;
655+
}
611656
this._decodeWaitForTxError(contractInstance, functionName, error, args);
612657
}
613658
return result;
614659
} catch (error) {
615660
const errorMessage = error.message.toLowerCase();
616661

617-
// Check for nonce-related errors
618-
if (
662+
const isNonceError =
619663
errorMessage.includes(
620664
EXPECTED_TRANSACTION_ERRORS.NONCE_TOO_LOW.toLowerCase(),
621665
) ||
622666
errorMessage.includes(
623667
EXPECTED_TRANSACTION_ERRORS.REPLACEMENT_UNDERPRICED.toLowerCase(),
624668
) ||
625-
errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.ALREADY_KNOWN.toLowerCase())
626-
) {
669+
errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.ALREADY_KNOWN.toLowerCase());
670+
671+
const isTimeoutError = errorMessage.includes(
672+
EXPECTED_TRANSACTION_ERRORS.TIMEOUT_EXCEEDED.toLowerCase(),
673+
);
674+
675+
const isExecutionError =
676+
errorMessage.includes(
677+
EXPECTED_TRANSACTION_ERRORS.EXECUTION_FAILED.toLowerCase(),
678+
) ||
679+
errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.FEE_TOO_LOW.toLowerCase());
680+
681+
const isNetworkError =
682+
errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.SOCKET_HANG_UP) ||
683+
errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.ECONNRESET) ||
684+
errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.ECONNREFUSED) ||
685+
errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.SERVER_ERROR) ||
686+
errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.BAD_GATEWAY) ||
687+
errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.SERVICE_UNAVAILABLE) ||
688+
errorMessage.includes(EXPECTED_TRANSACTION_ERRORS.EXPECT_BLOCK_NUMBER);
689+
690+
if (isNonceError || isTimeoutError || isExecutionError || isNetworkError) {
627691
retryCount += 1;
628692
if (retryCount < maxRetries) {
629-
// Increase gas price by 20% for nonce errors
630-
gasPrice = Math.ceil(gasPrice * 1.2);
693+
const shouldBumpGas = isNonceError || isExecutionError;
694+
if (shouldBumpGas) {
695+
gasPrice = ethers.BigNumber.isBigNumber(gasPrice)
696+
? gasPrice.mul(120).div(100)
697+
: Math.ceil(gasPrice * 1.2);
698+
}
699+
let errorType = 'Nonce';
700+
if (isTimeoutError) errorType = 'Timeout';
701+
else if (isExecutionError) errorType = 'Execution/fee';
702+
else if (isNetworkError) errorType = 'Network';
631703
this.logger.warn(
632-
`Nonce error detected for ${functionName}. Retrying with increased gas price: ${gasPrice} (retry ${retryCount}/${maxRetries})`,
704+
`${errorType} error detected for ${functionName} on ${this.getBlockchainId()}. ` +
705+
`Retrying ${
706+
shouldBumpGas
707+
? `with increased gas price: ${gasPrice}`
708+
: 'with same gas price'
709+
} (retry ${retryCount}/${maxRetries})`,
633710
);
634711
continue;
635-
} else {
636-
this.logger.error(
637-
`Max retries (${maxRetries}) reached for nonce error in ${functionName}. Final gas price: ${gasPrice}`,
638-
);
639712
}
713+
let errorType = 'nonce';
714+
if (isTimeoutError) errorType = 'timeout';
715+
else if (isExecutionError) errorType = 'execution/fee';
716+
else if (isNetworkError) errorType = 'network';
717+
this.logger.error(
718+
`Max retries (${maxRetries}) reached for ${errorType} error in ${functionName} on ${this.getBlockchainId()}. ` +
719+
`Final gas price: ${gasPrice}`,
720+
);
640721
}
641722

642-
// If it's not a nonce error or we've exhausted retries, re-throw the error
643723
throw error;
644724
}
645725
}

0 commit comments

Comments
 (0)