diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index f06492976..7547cf066 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -78,6 +78,14 @@ export const start = { default: 1.2, group: 'Ethereum', }) + .option('gas-limit-multiplier', { + description: + 'Multiplier applied to estimateGas results when computing transaction gas limits. ' + + 'Must be greater than 1.0. Default: 1.5', + type: 'number', + default: 1.5, + group: 'Ethereum', + }) .option('gas-price-max', { description: 'The maximum gas price (gwei) to use for transactions', type: 'number', @@ -356,9 +364,9 @@ export const start = { group: 'Indexer Infrastructure', }) .option('auto-allocation-max-batch-size', { - description: `Maximum number of allocation transactions inside a batch for auto allocation management. Limits the number of actions processed per batch to prevent multicall failures when there are many allocations. Remaining actions will be processed in subsequent batches.`, + description: `Maximum number of allocation transactions inside a batch. Upper bound is constrained by the block gas limit. Remaining actions will be processed in subsequent batches.`, type: 'number', - required: false, + default: 50, group: 'Indexer Infrastructure', }) .check(argv => { @@ -382,12 +390,21 @@ export const start = { if (argv['gas-increase-factor'] <= 1.0) { return 'Invalid --gas-increase-factor provided. Must be > 1.0' } + if (argv['gas-limit-multiplier'] <= 1.0) { + return 'Invalid --gas-limit-multiplier provided. Must be > 1.0' + } if ( !Number.isInteger(argv['rebate-claim-max-batch-size']) || argv['rebate-claim-max-batch-size'] <= 0 ) { return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.' } + if ( + !Number.isInteger(argv['auto-allocation-max-batch-size']) || + argv['auto-allocation-max-batch-size'] <= 0 + ) { + return 'Invalid --auto-allocation-max-batch-size provided. Must be > 0 and an integer.' + } return true }) }, @@ -433,6 +450,7 @@ export async function createNetworkSpecification( const transactionMonitoring = { gasIncreaseTimeout: argv.gasIncreaseTimeout, gasIncreaseFactor: argv.gasIncreaseFactor, + gasLimitMultiplier: argv.gasLimitMultiplier, gasPriceMax: argv.gasPriceMax, baseFeePerGasMax: argv.baseFeeGasMax, maxTransactionAttempts: argv.maxTransactionAttempts, diff --git a/packages/indexer-common/src/indexer-management/allocations.ts b/packages/indexer-common/src/indexer-management/allocations.ts index 5a363ceba..e917142e0 100644 --- a/packages/indexer-common/src/indexer-management/allocations.ts +++ b/packages/indexer-common/src/indexer-management/allocations.ts @@ -55,6 +55,7 @@ import { BigNumberish, BytesLike, ContractTransaction, + ContractTransactionResponse, hexlify, Result, TransactionReceipt, @@ -65,6 +66,32 @@ import { import pMap from 'p-map' import { tryParseCustomError } from '../utils' +// Known contract error selectors that indicate a logic failure, not gas exhaustion. +// These must NOT trigger batch bisection. +const KNOWN_CONTRACT_ERROR_SELECTORS = new Set([ + '0xd6bda275', // SubgraphServiceInvalidPaymentType(uint8) +]) + +function isGasExhaustionError(error: unknown): boolean { + if (!(error instanceof Error)) return false + const err = error as unknown as Record + if (err['code'] === 'CALL_EXCEPTION' && err['action'] === 'estimateGas') { + const data = err['data'] + if ( + typeof data === 'string' && + data.length >= 10 && + KNOWN_CONTRACT_ERROR_SELECTORS.has(data.slice(0, 10)) + ) { + return false + } + return err['reason'] == null + } + return ( + error.message.includes('gas required exceeds allowance') || + error.message.includes('exceeds block gas limit') + ) +} + export interface TransactionPreparationContext { activeAllocations: Allocation[] recentlyClosedAllocations: Allocation[] @@ -234,9 +261,10 @@ export class AllocationManager { ) // -- STAKING CONTRACT -- - const callDataStakingContract = stakingTransactions - .filter((tx: TransactionRequest) => !!tx.data) - .map((tx) => tx.data as string) + const eligibleStakingTxs = stakingTransactions.filter( + (tx: TransactionRequest) => !!tx.data, + ) + const callDataStakingContract = eligibleStakingTxs.map((tx) => tx.data as string) logger.debug('Found staking contract transactions', { count: callDataStakingContract.length, @@ -246,46 +274,24 @@ export class AllocationManager { }) if (callDataStakingContract.length > 0) { - try { - const stakingTransactionResult = - await this.network.transactionManager.executeTransaction( - async () => - this.network.contracts.HorizonStaking.multicall.estimateGas( - callDataStakingContract, - ), - async (gasLimit) => - this.network.contracts.HorizonStaking.multicall(callDataStakingContract, { - gasLimit, - }), - this.logger.child({ - actions: `${JSON.stringify(validatedActions.map((action) => action.id))}`, - function: 'staking.multicall', - }), - ) - - this.processActionResults( - actionResults, - stakingTransactions, - stakingTransactionResult, - ) - } catch (error) { - const parsedError = tryParseCustomError(error) - logger.error('Failed to execute staking contract transaction', { - error: parsedError, - }) - this.processActionResults(actionResults, stakingTransactions, { - failureReason: `Failed to execute staking contract transaction: ${ - typeof parsedError === 'string' ? parsedError : error.message - }`, - }) - } + await this.executeMulticallWithBisection( + callDataStakingContract, + eligibleStakingTxs, + actionResults, + this.logger.child({ + actions: `${JSON.stringify(validatedActions.map((action) => action.id))}`, + function: 'staking.multicall', + }), + (data) => this.network.contracts.HorizonStaking.multicall.estimateGas(data), + (data, gasLimit) => this.network.contracts.HorizonStaking.multicall(data, { gasLimit }), + ) } // -- SUBGRAPH SERVICE -- - const callDataSubgraphService = subgraphServiceTransactions - // Reallocate of a legacy allocation during the transition period can result in - // a staking and subgraph service transaction in the same batch. If the staking tx failed we - // should not execute the subgraph service tx. + // Reallocate of a legacy allocation during the transition period can result in + // a staking and subgraph service transaction in the same batch. If the staking tx failed we + // should not execute the subgraph service tx. + const eligibleSubgraphServiceTxs = subgraphServiceTransactions .filter((tx: ActionTransactionRequest) => { const actionStakingTransaction = actionResults.find( (result) => result.actionID === tx.actionID, @@ -296,7 +302,7 @@ export class AllocationManager { ) }) .filter((tx: TransactionRequest) => !!tx.data) - .map((tx) => tx.data as string) + const callDataSubgraphService = eligibleSubgraphServiceTxs.map((tx) => tx.data as string) logger.debug('Found subgraph service transactions', { count: callDataSubgraphService.length, }) @@ -305,39 +311,17 @@ export class AllocationManager { }) if (callDataSubgraphService.length > 0) { - try { - const subgraphServiceTransactionResult = - await this.network.transactionManager.executeTransaction( - async () => - this.network.contracts.SubgraphService.multicall.estimateGas( - callDataSubgraphService, - ), - async (gasLimit) => - this.network.contracts.SubgraphService.multicall(callDataSubgraphService, { - gasLimit, - }), - this.logger.child({ - actions: `${JSON.stringify(validatedActions.map((action) => action.id))}`, - function: 'subgraphService.multicall', - }), - ) - - this.processActionResults( - actionResults, - subgraphServiceTransactions, - subgraphServiceTransactionResult, - ) - } catch (error) { - const parsedError = tryParseCustomError(error) - logger.error('Failed to execute subgraph service transaction', { - error: parsedError, - }) - this.processActionResults(actionResults, subgraphServiceTransactions, { - failureReason: `Failed to execute subgraph service transaction: ${ - typeof parsedError === 'string' ? parsedError : error.message - }`, - }) - } + await this.executeMulticallWithBisection( + callDataSubgraphService, + eligibleSubgraphServiceTxs, + actionResults, + this.logger.child({ + actions: `${JSON.stringify(validatedActions.map((action) => action.id))}`, + function: 'subgraphService.multicall', + }), + (data) => this.network.contracts.SubgraphService.multicall.estimateGas(data), + (data, gasLimit) => this.network.contracts.SubgraphService.multicall(data, { gasLimit }), + ) } // sanity check that all actions have a result @@ -353,6 +337,55 @@ export class AllocationManager { return actionResults } + private async executeMulticallWithBisection( + callData: string[], + txs: ActionTransactionRequest[], + actionResults: ExecuteActionResult[], + logger: Logger, + estimateFn: (callData: string[]) => Promise, + submitFn: (callData: string[], gasLimit: BigNumberish) => Promise, + ): Promise { + try { + const result = await this.network.transactionManager.executeTransaction( + () => estimateFn(callData), + (gasLimit) => submitFn(callData, gasLimit), + logger, + ) + this.processActionResults(actionResults, txs, result) + } catch (error) { + if (isGasExhaustionError(error) && callData.length > 1) { + const mid = Math.floor(callData.length / 2) + logger.warn(`Multicall gas estimation failed for batch of ${callData.length}, bisecting`, { + batchSize: callData.length, + }) + await this.executeMulticallWithBisection( + callData.slice(0, mid), + txs.slice(0, mid), + actionResults, + logger, + estimateFn, + submitFn, + ) + await this.executeMulticallWithBisection( + callData.slice(mid), + txs.slice(mid), + actionResults, + logger, + estimateFn, + submitFn, + ) + } else { + const parsedError = tryParseCustomError(error) + logger.error('Failed to execute multicall transaction', { error: parsedError }) + this.processActionResults(actionResults, txs, { + failureReason: `Failed to execute multicall transaction: ${ + typeof parsedError === 'string' ? parsedError : (error as Error).message + }`, + }) + } + } + } + /** * Processes the result of transaction batches for an action * diff --git a/packages/indexer-common/src/network-specification.ts b/packages/indexer-common/src/network-specification.ts index 7bb86d6e2..b6a3c9f45 100644 --- a/packages/indexer-common/src/network-specification.ts +++ b/packages/indexer-common/src/network-specification.ts @@ -62,7 +62,7 @@ export const IndexerOptions = z .default('auto') .transform((x) => x as AllocationManagementMode), autoAllocationMinBatchSize: positiveNumber().default(1), - autoAllocationMaxBatchSize: positiveNumber().optional(), + autoAllocationMaxBatchSize: positiveNumber().default(50), allocateOnNetworkSubgraph: z.boolean().default(false), register: z.boolean().default(true), maxProvisionInitialSize: GRT() @@ -83,6 +83,7 @@ export const TransactionMonitoring = z .default(240) .transform((x) => x * 10 ** 3), gasIncreaseFactor: positiveNumber().default(1.2), + gasLimitMultiplier: positiveNumber().default(1.5), gasPriceMax: positiveNumber() .default(100) .transform((x) => x * 10 ** 9), diff --git a/packages/indexer-common/src/transactions.ts b/packages/indexer-common/src/transactions.ts index fc99bbf20..77c6d7233 100644 --- a/packages/indexer-common/src/transactions.ts +++ b/packages/indexer-common/src/transactions.ts @@ -73,7 +73,9 @@ export class TransactionManager { let output: TransactionReceipt | undefined = undefined const feeData = await this.waitForGasPricesBelowThreshold(logger) - const paddedGasLimit = Math.ceil(Number(await gasEstimation()) * 1.5) + const paddedGasLimit = Math.ceil( + Number(await gasEstimation()) * this.specification.gasLimitMultiplier, + ) const txPromise = transaction(paddedGasLimit) let tx: TransactionResponse = await txPromise