fix: configurable allocation batch size, gas limit multiplier, and multicall bisection

packages/indexer-agent/src/commands/start.ts

@@ -78,6 +78,14 @@ export const start = {
         default: 1.2,
         group: 'Ethereum',
       })
+      .option('gas-limit-multiplier', {
+        description:
+          'Multiplier applied to estimateGas results when computing transaction gas limits. ' +
+          'Must be greater than 1.0. Default: 1.5',
+        type: 'number',
+        default: 1.5,
+        group: 'Ethereum',
+      })
       .option('gas-price-max', {
         description: 'The maximum gas price (gwei) to use for transactions',
         type: 'number',
@@ -356,9 +364,9 @@ export const start = {
         group: 'Indexer Infrastructure',
       })
       .option('auto-allocation-max-batch-size', {
-        description: `Maximum number of allocation transactions inside a batch for auto allocation management. Limits the number of actions processed per batch to prevent multicall failures when there are many allocations. Remaining actions will be processed in subsequent batches.`,
+        description: `Maximum number of allocation transactions inside a batch. Upper bound is constrained by the block gas limit. Remaining actions will be processed in subsequent batches.`,
         type: 'number',
-        required: false,
+        default: 50,
         group: 'Indexer Infrastructure',
       })
       .check(argv => {
@@ -382,12 +390,21 @@ export const start = {
         if (argv['gas-increase-factor'] <= 1.0) {
           return 'Invalid --gas-increase-factor provided. Must be > 1.0'
         }
+        if (argv['gas-limit-multiplier'] <= 1.0) {
+          return 'Invalid --gas-limit-multiplier provided. Must be > 1.0'
+        }
         if (
           !Number.isInteger(argv['rebate-claim-max-batch-size']) ||
           argv['rebate-claim-max-batch-size'] <= 0
         ) {
           return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.'
         }
+        if (
+          !Number.isInteger(argv['auto-allocation-max-batch-size']) ||
+          argv['auto-allocation-max-batch-size'] <= 0
+        ) {
+          return 'Invalid --auto-allocation-max-batch-size provided. Must be > 0 and an integer.'
+        }
         return true
       })
   },
@@ -433,6 +450,7 @@ export async function createNetworkSpecification(
   const transactionMonitoring = {
     gasIncreaseTimeout: argv.gasIncreaseTimeout,
     gasIncreaseFactor: argv.gasIncreaseFactor,
+    gasLimitMultiplier: argv.gasLimitMultiplier,
     gasPriceMax: argv.gasPriceMax,
     baseFeePerGasMax: argv.baseFeeGasMax,
     maxTransactionAttempts: argv.maxTransactionAttempts,

packages/indexer-common/src/indexer-management/allocations.ts

@@ -55,6 +55,7 @@ import {
   BigNumberish,
   BytesLike,
   ContractTransaction,
+  ContractTransactionResponse,
   hexlify,
   Result,
   TransactionReceipt,
@@ -65,6 +66,32 @@ import {
 import pMap from 'p-map'
 import { tryParseCustomError } from '../utils'
 
+// Known contract error selectors that indicate a logic failure, not gas exhaustion.
+// These must NOT trigger batch bisection.
+const KNOWN_CONTRACT_ERROR_SELECTORS = new Set([
+  '0xd6bda275', // SubgraphServiceInvalidPaymentType(uint8)
+])
+
+function isGasExhaustionError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false
+  const err = error as unknown as Record<string, unknown>
+  if (err['code'] === 'CALL_EXCEPTION' && err['action'] === 'estimateGas') {
+    const data = err['data']
+    if (
+      typeof data === 'string' &&
+      data.length >= 10 &&
+      KNOWN_CONTRACT_ERROR_SELECTORS.has(data.slice(0, 10))
+    ) {
+      return false
+    }
+    return err['reason'] == null
+  }
+  return (
+    error.message.includes('gas required exceeds allowance') ||
+    error.message.includes('exceeds block gas limit')
+  )
+}
+
 export interface TransactionPreparationContext {
   activeAllocations: Allocation[]
   recentlyClosedAllocations: Allocation[]
@@ -234,9 +261,10 @@ export class AllocationManager {
     )
 
     // -- STAKING CONTRACT --
-    const callDataStakingContract = stakingTransactions
-      .filter((tx: TransactionRequest) => !!tx.data)
-      .map((tx) => tx.data as string)
+    const eligibleStakingTxs = stakingTransactions.filter(
+      (tx: TransactionRequest) => !!tx.data,
+    )
+    const callDataStakingContract = eligibleStakingTxs.map((tx) => tx.data as string)
 
     logger.debug('Found staking contract transactions', {
       count: callDataStakingContract.length,
@@ -246,46 +274,24 @@ export class AllocationManager {
     })
 
     if (callDataStakingContract.length > 0) {
-      try {
-        const stakingTransactionResult =
-          await this.network.transactionManager.executeTransaction(
-            async () =>
-              this.network.contracts.HorizonStaking.multicall.estimateGas(
-                callDataStakingContract,
-              ),
-            async (gasLimit) =>
-              this.network.contracts.HorizonStaking.multicall(callDataStakingContract, {
-                gasLimit,
-              }),
-            this.logger.child({
-              actions: `${JSON.stringify(validatedActions.map((action) => action.id))}`,
-              function: 'staking.multicall',
-            }),
-          )
-
-        this.processActionResults(
-          actionResults,
-          stakingTransactions,
-          stakingTransactionResult,
-        )
-      } catch (error) {
-        const parsedError = tryParseCustomError(error)
-        logger.error('Failed to execute staking contract transaction', {
-          error: parsedError,
-        })
-        this.processActionResults(actionResults, stakingTransactions, {
-          failureReason: `Failed to execute staking contract transaction: ${
-            typeof parsedError === 'string' ? parsedError : error.message
-          }`,
-        })
-      }
+      await this.executeMulticallWithBisection(
+        callDataStakingContract,
+        eligibleStakingTxs,
+        actionResults,
+        this.logger.child({
+          actions: `${JSON.stringify(validatedActions.map((action) => action.id))}`,
+          function: 'staking.multicall',
+        }),
+        (data) => this.network.contracts.HorizonStaking.multicall.estimateGas(data),
+        (data, gasLimit) => this.network.contracts.HorizonStaking.multicall(data, { gasLimit }),
+      )
     }
 
     // -- SUBGRAPH SERVICE --
-    const callDataSubgraphService = subgraphServiceTransactions
-      // Reallocate of a legacy allocation during the transition period can result in
-      // a staking and subgraph service transaction in the same batch. If the staking tx failed we
-      // should not execute the subgraph service tx.
+    // Reallocate of a legacy allocation during the transition period can result in
+    // a staking and subgraph service transaction in the same batch. If the staking tx failed we
+    // should not execute the subgraph service tx.
+    const eligibleSubgraphServiceTxs = subgraphServiceTransactions
       .filter((tx: ActionTransactionRequest) => {
         const actionStakingTransaction = actionResults.find(
           (result) => result.actionID === tx.actionID,
@@ -296,7 +302,7 @@ export class AllocationManager {
         )
       })
       .filter((tx: TransactionRequest) => !!tx.data)
-      .map((tx) => tx.data as string)
+    const callDataSubgraphService = eligibleSubgraphServiceTxs.map((tx) => tx.data as string)
     logger.debug('Found subgraph service transactions', {
       count: callDataSubgraphService.length,
     })
@@ -305,39 +311,17 @@ export class AllocationManager {
     })
 
     if (callDataSubgraphService.length > 0) {
-      try {
-        const subgraphServiceTransactionResult =
-          await this.network.transactionManager.executeTransaction(
-            async () =>
-              this.network.contracts.SubgraphService.multicall.estimateGas(
-                callDataSubgraphService,
-              ),
-            async (gasLimit) =>
-              this.network.contracts.SubgraphService.multicall(callDataSubgraphService, {
-                gasLimit,
-              }),
-            this.logger.child({
-              actions: `${JSON.stringify(validatedActions.map((action) => action.id))}`,
-              function: 'subgraphService.multicall',
-            }),
-          )
-
-        this.processActionResults(
-          actionResults,
-          subgraphServiceTransactions,
-          subgraphServiceTransactionResult,
-        )
-      } catch (error) {
-        const parsedError = tryParseCustomError(error)
-        logger.error('Failed to execute subgraph service transaction', {
-          error: parsedError,
-        })
-        this.processActionResults(actionResults, subgraphServiceTransactions, {
-          failureReason: `Failed to execute subgraph service transaction: ${
-            typeof parsedError === 'string' ? parsedError : error.message
-          }`,
-        })
-      }
+      await this.executeMulticallWithBisection(
+        callDataSubgraphService,
+        eligibleSubgraphServiceTxs,
+        actionResults,
+        this.logger.child({
+          actions: `${JSON.stringify(validatedActions.map((action) => action.id))}`,
+          function: 'subgraphService.multicall',
+        }),
+        (data) => this.network.contracts.SubgraphService.multicall.estimateGas(data),
+        (data, gasLimit) => this.network.contracts.SubgraphService.multicall(data, { gasLimit }),
+      )
     }
 
     // sanity check that all actions have a result
@@ -353,6 +337,55 @@ export class AllocationManager {
     return actionResults
   }
 
+  private async executeMulticallWithBisection(
+    callData: string[],
+    txs: ActionTransactionRequest[],
+    actionResults: ExecuteActionResult[],
+    logger: Logger,
+    estimateFn: (callData: string[]) => Promise<bigint>,
+    submitFn: (callData: string[], gasLimit: BigNumberish) => Promise<ContractTransactionResponse>,
+  ): Promise<void> {
+    try {
+      const result = await this.network.transactionManager.executeTransaction(
+        () => estimateFn(callData),
+        (gasLimit) => submitFn(callData, gasLimit),
+        logger,
+      )
+      this.processActionResults(actionResults, txs, result)
+    } catch (error) {
+      if (isGasExhaustionError(error) && callData.length > 1) {
+        const mid = Math.floor(callData.length / 2)
+        logger.warn(`Multicall gas estimation failed for batch of ${callData.length}, bisecting`, {
+          batchSize: callData.length,
+        })
+        await this.executeMulticallWithBisection(
+          callData.slice(0, mid),
+          txs.slice(0, mid),
+          actionResults,
+          logger,
+          estimateFn,
+          submitFn,
+        )
+        await this.executeMulticallWithBisection(
+          callData.slice(mid),
+          txs.slice(mid),
+          actionResults,
+          logger,
+          estimateFn,
+          submitFn,
+        )
+      } else {
+        const parsedError = tryParseCustomError(error)
+        logger.error('Failed to execute multicall transaction', { error: parsedError })
+        this.processActionResults(actionResults, txs, {
+          failureReason: `Failed to execute multicall transaction: ${
+            typeof parsedError === 'string' ? parsedError : (error as Error).message
+          }`,
+        })
+      }
+    }
+  }
+
   /**
    * Processes the result of transaction batches for an action
    *

packages/indexer-common/src/network-specification.ts

@@ -62,7 +62,7 @@ export const IndexerOptions = z
       .default('auto')
       .transform((x) => x as AllocationManagementMode),
     autoAllocationMinBatchSize: positiveNumber().default(1),
-    autoAllocationMaxBatchSize: positiveNumber().optional(),
+    autoAllocationMaxBatchSize: positiveNumber().default(50),
     allocateOnNetworkSubgraph: z.boolean().default(false),
     register: z.boolean().default(true),
     maxProvisionInitialSize: GRT()
@@ -83,6 +83,7 @@ export const TransactionMonitoring = z
       .default(240)
       .transform((x) => x * 10 ** 3),
     gasIncreaseFactor: positiveNumber().default(1.2),
+    gasLimitMultiplier: positiveNumber().default(1.5),
     gasPriceMax: positiveNumber()
       .default(100)
       .transform((x) => x * 10 ** 9),

packages/indexer-common/src/transactions.ts

@@ -73,7 +73,9 @@ export class TransactionManager {
     let output: TransactionReceipt | undefined = undefined
 
     const feeData = await this.waitForGasPricesBelowThreshold(logger)
-    const paddedGasLimit = Math.ceil(Number(await gasEstimation()) * 1.5)
+    const paddedGasLimit = Math.ceil(
+      Number(await gasEstimation()) * this.specification.gasLimitMultiplier,
+    )
 
     const txPromise = transaction(paddedGasLimit)
     let tx: TransactionResponse = await txPromise