fix: gracefully handle overallocation when reallocating (#1203)

Signed-off-by: Tomás Migone <tomas@edgeandnode.com>
Co-authored-by: MoonBoi9001 <67825802+MoonBoi9001@users.noreply.github.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: 3 people

packages/indexer-common/src/errors.ts

@@ -100,6 +100,7 @@ export enum IndexerErrorCode {
   IE087 = 'IE087',
   IE088 = 'IE088',
   IE089 = 'IE089',
+  IE090 = 'IE090',
 }
 
 export const INDEXER_ERROR_MESSAGES: Record<IndexerErrorCode, string> = {
@@ -193,6 +194,7 @@ export const INDEXER_ERROR_MESSAGES: Record<IndexerErrorCode, string> = {
   IE087: 'Failed to resize allocation',
   IE088: 'Failed to present POI',
   IE089: 'Failed to collect indexing rewards',
+  IE090: 'Failed to reallocate: indexer is overallocated',
 }
 
 export type IndexerErrorCause = unknown

packages/indexer-common/src/indexer-management/__tests__/over-allocation.test.ts

@@ -0,0 +1,159 @@
+import { createLogger, Logger } from '@graphprotocol/common-ts'
+import { assertNotOverAllocated } from '../over-allocation'
+import { IndexerError, IndexerErrorCode } from '../../errors'
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+declare const __LOG_LEVEL__: any
+
+const indexer = '0x0000000000000000000000000000000000000001'
+const allocationId = '0x000000000000000000000000000000000000000a'
+const subgraphServiceAddress = '0x00000000000000000000000000000000000000ff'
+
+interface MockSubgraphService {
+  isOverAllocated: jest.Mock
+  allocationProvisionTracker: jest.Mock
+  getDelegationRatio: jest.Mock
+  target: string
+}
+
+interface MockHorizonStaking {
+  getTokensAvailable: jest.Mock
+}
+
+const buildContracts = (
+  overrides: {
+    isOverAllocated?: boolean
+    allocatedTokens?: bigint
+    delegationRatio?: bigint
+    tokensAvailable?: bigint
+  } = {},
+): {
+  contracts: { SubgraphService: MockSubgraphService; HorizonStaking: MockHorizonStaking }
+  subgraphService: MockSubgraphService
+  horizonStaking: MockHorizonStaking
+} => {
+  const subgraphService: MockSubgraphService = {
+    isOverAllocated: jest.fn().mockResolvedValue(overrides.isOverAllocated ?? false),
+    allocationProvisionTracker: jest
+      .fn()
+      .mockResolvedValue(overrides.allocatedTokens ?? 0n),
+    getDelegationRatio: jest.fn().mockResolvedValue(overrides.delegationRatio ?? 1n),
+    target: subgraphServiceAddress,
+  }
+  const horizonStaking: MockHorizonStaking = {
+    getTokensAvailable: jest.fn().mockResolvedValue(overrides.tokensAvailable ?? 0n),
+  }
+  return {
+    contracts: { SubgraphService: subgraphService, HorizonStaking: horizonStaking },
+    subgraphService,
+    horizonStaking,
+  }
+}
+
+describe('assertNotOverAllocated', () => {
+  let logger: Logger
+
+  beforeAll(() => {
+    logger = createLogger({
+      name: 'over-allocation-test',
+      async: false,
+      level: __LOG_LEVEL__ ?? 'error',
+    })
+  })
+
+  it('returns without throwing when the indexer is not over-allocated', async () => {
+    const { contracts, subgraphService, horizonStaking } = buildContracts({
+      isOverAllocated: false,
+    })
+
+    await expect(
+      assertNotOverAllocated(
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        contracts as any,
+        indexer,
+        logger,
+        allocationId,
+      ),
+    ).resolves.toBeUndefined()
+
+    expect(subgraphService.isOverAllocated).toHaveBeenCalledWith(indexer)
+    // None of the diagnostic reads should fire on the happy path — they are
+    // only needed to compose the error message when over-allocation is true.
+    expect(subgraphService.allocationProvisionTracker).not.toHaveBeenCalled()
+    expect(subgraphService.getDelegationRatio).not.toHaveBeenCalled()
+    expect(horizonStaking.getTokensAvailable).not.toHaveBeenCalled()
+  })
+
+  it('throws IE090 with the over-allocated delta when over-allocated', async () => {
+    // 1000 GRT allocated, 600 GRT available => 400 GRT over.
+    const { contracts } = buildContracts({
+      isOverAllocated: true,
+      allocatedTokens: 1000n * 10n ** 18n,
+      delegationRatio: 2n,
+      tokensAvailable: 600n * 10n ** 18n,
+    })
+
+    let captured: unknown
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      await assertNotOverAllocated(contracts as any, indexer, logger, allocationId)
+    } catch (err) {
+      captured = err
+    }
+
+    expect(captured).toBeInstanceOf(IndexerError)
+    const err = captured as IndexerError
+    expect(err.code).toBe(IndexerErrorCode.IE090)
+    // Spot-check the cause string contains the GRT delta and the actionable
+    // hint pointing operators at the close path (which still collects rewards).
+    const cause = String(err.cause)
+    expect(cause).toContain('400.0')
+    expect(cause).toContain('graph indexer allocations close')
+  })
+
+  it('passes the SubgraphService address as the verifier to getTokensAvailable', async () => {
+    const { contracts, horizonStaking } = buildContracts({
+      isOverAllocated: true,
+      allocatedTokens: 100n,
+      delegationRatio: 3n,
+      tokensAvailable: 50n,
+    })
+
+    await expect(
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      assertNotOverAllocated(contracts as any, indexer, logger, allocationId),
+    ).rejects.toBeInstanceOf(IndexerError)
+
+    expect(horizonStaking.getTokensAvailable).toHaveBeenCalledWith(
+      indexer,
+      subgraphServiceAddress,
+      3n,
+    )
+  })
+
+  it('clamps the over-allocated amount to zero if reads race to a negative delta', async () => {
+    // tokensAvailable > allocatedTokens shouldn't happen when isOverAllocated
+    // is true, but the three reads aren't atomic; defend against a negative
+    // bigint formatting into the error message.
+    const { contracts } = buildContracts({
+      isOverAllocated: true,
+      allocatedTokens: 100n,
+      delegationRatio: 1n,
+      tokensAvailable: 200n,
+    })
+
+    let captured: unknown
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      await assertNotOverAllocated(contracts as any, indexer, logger, allocationId)
+    } catch (err) {
+      captured = err
+    }
+
+    const err = captured as IndexerError
+    expect(err.code).toBe(IndexerErrorCode.IE090)
+    const cause = String(err.cause)
+    expect(cause).toContain('0.0 GRT')
+    expect(cause).not.toContain('-')
+  })
+})

packages/indexer-common/src/indexer-management/allocations.ts

@@ -50,6 +50,7 @@ import {
   encodeCollectIndexingRewardsData,
   encodePOIMetadata,
 } from '@graphprotocol/toolshed'
+import { assertNotOverAllocated } from './over-allocation'
 
 import {
   BigNumberish,
@@ -2123,6 +2124,13 @@ export class AllocationManager {
         }
       }
     } else {
+      await assertNotOverAllocated(
+        this.network.contracts,
+        params.indexer,
+        logger,
+        params.closingAllocationID,
+      )
+
       // Horizon: Need to multicall collect and stopService
 
       // collect

packages/indexer-common/src/indexer-management/index.ts

@@ -3,6 +3,7 @@ export * from './allocations'
 export * from './client'
 export * from './models'
 export * from './monitor'
+export * from './over-allocation'
 export * from './server'
 export * from './rules'
 export * from './types'

packages/indexer-common/src/indexer-management/over-allocation.ts

@@ -0,0 +1,48 @@
+import { formatGRT, Logger } from '@graphprotocol/common-ts'
+import {
+  GraphHorizonContracts,
+  SubgraphServiceContracts,
+} from '@graphprotocol/toolshed/deployments'
+import { indexerError, IndexerErrorCode } from '../errors'
+
+// Throws IE090 if the indexer is over-allocated on the SubgraphService.
+// Used by reallocate paths: collect() would auto-close the existing allocation
+// and the new allocation would be rejected, leaving the indexer with no
+// allocation on the deployment. The indexer can still collect rewards by
+// closing the allocation directly via `graph indexer allocations close`,
+// which handles over-allocation gracefully.
+export async function assertNotOverAllocated(
+  contracts: GraphHorizonContracts & SubgraphServiceContracts,
+  indexer: string,
+  logger: Logger,
+  allocationId: string,
+): Promise<void> {
+  const isOverAllocated = await contracts.SubgraphService.isOverAllocated(indexer)
+
+  logger.debug('Checking over-allocation status for reallocate allocation', {
+    allocationId,
+    isOverAllocated,
+  })
+
+  if (!isOverAllocated) {
+    return
+  }
+
+  const [allocatedTokens, delegationRatio] = await Promise.all([
+    contracts.SubgraphService.allocationProvisionTracker(indexer),
+    contracts.SubgraphService.getDelegationRatio(),
+  ])
+  const tokensAvailable = await contracts.HorizonStaking.getTokensAvailable(
+    indexer,
+    contracts.SubgraphService.target,
+    delegationRatio,
+  )
+  const overallocatedAmount =
+    allocatedTokens > tokensAvailable ? allocatedTokens - tokensAvailable : 0n
+  throw indexerError(
+    IndexerErrorCode.IE090,
+    `Overallocated by ${formatGRT(
+      overallocatedAmount,
+    )} GRT. Close this allocation via 'graph indexer allocations close' to collect rewards, or add provision tokens before retrying reallocate.`,
+  )
+}

packages/indexer-common/src/indexer-management/resolvers/allocations.ts

@@ -16,6 +16,7 @@ import {
 import {
   Allocation,
   AllocationStatus,
+  assertNotOverAllocated,
   CloseAllocationResult,
   CreateAllocationResult,
   encodeCollectData,
@@ -1207,6 +1208,8 @@ async function reallocateHorizonAllocation(
     epoch: currentEpoch.toString(),
   })
 
+  await assertNotOverAllocated(contracts, address, logger, allocation.id)
+
   // Identify how many GRT the indexer has staked
   const freeStake = (await network.networkMonitor.freeStake()).horizon