feat: configurable max block range for EventFetcher chunking

packages/sdk/src/index.ts

@@ -79,7 +79,7 @@ export {
   mapL2NetworkTokenBridgeToTokenBridge,
 } from './lib/dataEntities/networks'
 export { InboxTools } from './lib/inbox/inbox'
-export { EventFetcher } from './lib/utils/eventFetcher'
+export { EventFetcher, DEFAULT_MAX_BLOCK_RANGE } from './lib/utils/eventFetcher'
 export { ArbitrumProvider } from './lib/utils/arbProvider'
 export * as constants from './lib/dataEntities/constants'
 export {

packages/sdk/src/lib/utils/eventFetcher.ts

@@ -16,12 +16,20 @@
 /* eslint-env node */
 'use strict'
 
-import { Provider, BlockTag, Filter } from '@ethersproject/abstract-provider'
+import {
+  Provider,
+  BlockTag,
+  Filter,
+  Log,
+} from '@ethersproject/abstract-provider'
 import { Contract, Event } from '@ethersproject/contracts'
 import { constants } from 'ethers'
 import { TypedEvent, TypedEventFilter } from '../abi/common'
 import { EventArgs, TypeChainContractFactory } from '../dataEntities/event'
 
+export const DEFAULT_MAX_BLOCK_RANGE = 10_000
+const MIN_CHUNK_SIZE = 500
+
 export type FetchedEvent<TEvent extends Event> = {
   event: EventArgs<TEvent>
   topic: string
@@ -45,6 +53,16 @@ type TEventOf<T> = T extends TypedEventFilter<infer TEvent> ? TEvent : never
  * Fetches and parses blockchain logs
  */
 export class EventFetcher {
+  private static maxBlockRange: number = DEFAULT_MAX_BLOCK_RANGE
+
+  /**
+   * Set the maximum block range used by all EventFetcher instances
+   * when chunking log queries after an initial failure.
+   */
+  public static setMaxBlockRange(maxBlockRange: number): void {
+    EventFetcher.maxBlockRange = maxBlockRange
+  }
+
   public constructor(public readonly provider: Provider) {}
 
   /**
@@ -77,7 +95,7 @@ export class EventFetcher {
       fromBlock: filter.fromBlock,
       toBlock: filter.toBlock,
     }
-    const logs = await this.provider.getLogs(fullFilter)
+    const logs = await this.getLogsWithChunking(fullFilter)
     return logs
       .filter(l => l.removed === false)
       .map(l => {
@@ -97,4 +115,88 @@ export class EventFetcher {
         }
       }) as FetchedEvent<TEventOf<TEventFilter>>[]
   }
+
+  private async resolveBlockTag(
+    blockTag: BlockTag | undefined
+  ): Promise<number | null> {
+    if (typeof blockTag === 'number') {
+      if (blockTag >= 0) return blockTag
+
+      const latestBlock = await this.provider.getBlockNumber()
+      return Math.max(latestBlock + blockTag, 0)
+    }
+    if (blockTag === 'earliest') return 0
+    if (typeof blockTag === 'string' && blockTag.startsWith('0x')) {
+      const parsed = parseInt(blockTag, 16)
+      return isNaN(parsed) ? null : parsed
+    }
+    // 'latest', 'pending', 'safe', 'finalized', undefined
+    const block = await this.provider.getBlock(blockTag ?? 'latest')
+    return block?.number ?? null
+  }
+
+  private async getLogsWithChunking(filter: Filter): Promise<Log[]> {
+    try {
+      return await this.provider.getLogs(filter)
+    } catch (error) {
+      const fromBlock = await this.resolveBlockTag(filter.fromBlock)
+      const toBlock = await this.resolveBlockTag(filter.toBlock)
+
+      if (fromBlock === null || toBlock === null) {
+        throw error
+      }
+
+      const initialChunkSize = Math.min(
+        EventFetcher.maxBlockRange,
+        toBlock - fromBlock + 1
+      )
+
+      if (initialChunkSize <= 0) {
+        throw error
+      }
+
+      return this.fetchLogsInRange(filter, fromBlock, toBlock, initialChunkSize)
+    }
+  }
+
+  private async fetchLogsInRange(
+    filter: Filter,
+    fromBlock: number,
+    toBlock: number,
+    chunkSize: number
+  ): Promise<Log[]> {
+    const allLogs: Log[] = []
+    let start = fromBlock
+
+    while (start <= toBlock) {
+      if (start > fromBlock) {
+        await new Promise(r => setTimeout(r, 100))
+      }
+
+      const end = Math.min(start + chunkSize - 1, toBlock)
+      try {
+        const chunkLogs = await this.provider.getLogs({
+          ...filter,
+          fromBlock: start,
+          toBlock: end,
+        })
+        allLogs.push(...chunkLogs)
+      } catch (error) {
+        if (chunkSize > MIN_CHUNK_SIZE) {
+          const retryLogs = await this.fetchLogsInRange(
+            filter,
+            start,
+            end,
+            Math.floor(chunkSize / 2)
+          )
+          allLogs.push(...retryLogs)
+        } else {
+          throw error
+        }
+      }
+      start = end + 1
+    }
+
+    return allLogs
+  }
 }