release branch for v0.25.3 (#1149)

Author: dwerner

.gitignore

@@ -123,4 +123,6 @@ yalc.lock
 
 # local-network override mode files
 tap-contracts.json
-config/config.yaml
+config/config.yaml
+
+commit-changes.sh

packages/indexer-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graphprotocol/indexer-agent",
-  "version": "0.25.2",
+  "version": "0.25.3",
   "description": "Indexer agent",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -30,7 +30,7 @@
   },
   "dependencies": {
     "@graphprotocol/common-ts": "3.0.1",
-    "@graphprotocol/indexer-common": "0.25.2",
+    "@graphprotocol/indexer-common": "0.25.3",
     "@thi.ng/heaps": "^1.3.1",
     "axios": "0.26.1",
     "bs58": "5.0.0",

packages/indexer-agent/src/agent.ts

@@ -8,7 +8,6 @@ import {
   timer,
 } from '@graphprotocol/common-ts'
 import {
-  ActivationCriteria,
   ActionStatus,
   Allocation,
   AllocationManagementMode,
@@ -32,9 +31,6 @@ import {
   validateProviderNetworkIdentifier,
   MultiNetworks,
   NetworkMapped,
-  TransferredSubgraphDeployment,
-  networkIsL2,
-  networkIsL1,
   DeploymentManagementMode,
   SubgraphStatus,
   sequentialTimerMap,
@@ -44,7 +40,6 @@ import {
 import PQueue from 'p-queue'
 import pMap from 'p-map'
 import pFilter from 'p-filter'
-import mapValues from 'lodash.mapvalues'
 import zip from 'lodash.zip'
 import { AgentConfigs, NetworkAndOperator } from './types'
 
@@ -194,7 +189,6 @@ export class Agent {
   multiNetworks: MultiNetworks<NetworkAndOperator>
   indexerManagement: IndexerManagementClient
   offchainSubgraphs: SubgraphDeploymentID[]
-  autoMigrationSupport: boolean
   deploymentManagement: DeploymentManagementMode
   pollingInterval: number
 
@@ -208,7 +202,6 @@ export class Agent {
       configs.operators,
     )
     this.offchainSubgraphs = configs.offchainSubgraphs
-    this.autoMigrationSupport = !!configs.autoMigrationSupport
     this.deploymentManagement = configs.deploymentManagement
     this.pollingInterval = configs.pollingInterval
   }
@@ -377,69 +370,36 @@ export class Agent {
         },
       )
 
-    const eligibleTransferDeployments: Eventual<
-      NetworkMapped<TransferredSubgraphDeployment[]>
-    > = sequentialTimerMap(
-      { logger, milliseconds: requestIntervalLarge },
-      async () => {
-        // Return early if the auto migration feature is disabled.
-        if (!this.autoMigrationSupport) {
-          logger.trace(
-            'Auto Migration feature is disabled, skipping querying transferred subgraphs',
-          )
-          return this.multiNetworks.map(async () => [])
-        }
-
-        const statuses = await this.graphNode.indexingStatus([])
-        return this.multiNetworks.map(async ({ network }) => {
-          const protocolNetwork = network.specification.networkIdentifier
-          logger.trace('Fetching deployments eligible for L2 transfer', {
-            protocolNetwork,
-          })
-          const transfers =
-            await network.networkMonitor.transferredDeployments()
-          logger.trace(
-            `Found ${transfers.length} transferred subgraphs in the network`,
-            { protocolNetwork },
-          )
-          return transfers
-            .map(transfer => {
-              const status = statuses.find(
-                status =>
-                  status.subgraphDeployment.ipfsHash == transfer.ipfsHash,
-              )
-              if (status) {
-                transfer.ready = status.synced && status.health == 'healthy'
-              }
-              return transfer
-            })
-            .filter(transfer => transfer.ready == true)
-        })
-      },
-      {
-        onError: error =>
-          logger.warn(
-            `Failed to obtain transferred deployments, trying again later`,
-            { error },
-          ),
-      },
-    )
-
-    // While in the L1 -> L2 transfer period this will be an intermediate value
-    // with the final value including transfer considerations
-    const intermediateNetworkDeploymentAllocationDecisions: Eventual<
+    const networkDeploymentAllocationDecisions: Eventual<
       NetworkMapped<AllocationDecision[]>
     > = join({
       networkDeployments,
       indexingRules,
     }).tryMap(
-      ({ indexingRules, networkDeployments }) => {
-        return mapValues(
+      async ({ indexingRules, networkDeployments }) => {
+        return this.multiNetworks.mapNetworkMapped(
           this.multiNetworks.zip(indexingRules, networkDeployments),
-          ([indexingRules, networkDeployments]: [
-            IndexingRuleAttributes[],
-            SubgraphDeployment[],
-          ]) => {
+          async (
+            { network }: NetworkAndOperator,
+            [indexingRules, networkDeployments]: [
+              IndexingRuleAttributes[],
+              SubgraphDeployment[],
+            ],
+          ) => {
+            // Skip evaluation entirely for networks in manual allocation mode
+            if (
+              network.specification.indexerOptions.allocationManagementMode ===
+              AllocationManagementMode.MANUAL
+            ) {
+              logger.trace(
+                `Skipping deployment evaluation since AllocationManagementMode = 'manual'`,
+                {
+                  protocolNetwork: network.specification.networkIdentifier,
+                },
+              )
+              return []
+            }
+
             // Identify subgraph deployments on the network that are worth picking up;
             // these may overlap with the ones we're already indexing
             logger.trace('Evaluating which deployments are worth allocating to')
@@ -457,94 +417,6 @@ export class Agent {
       },
     )
 
-    // Update targetDeployments and networkDeplomentAllocationDecisions using transferredSubgraphDeployments data
-    // This will be somewhat custom and will likely be yanked out later after the transfer stage is complete
-    // Cases:
-    // - L1 subgraph that had the transfer started: keep synced and allocated to for at least one week
-    //   post transfer.
-    // - L2 subgraph that has been transferred:
-    //   - if already synced, allocate to it immediately using default allocation amount
-    //   - if not synced, no changes
-    const networkDeploymentAllocationDecisions: Eventual<
-      NetworkMapped<AllocationDecision[]>
-    > = join({
-      intermediateNetworkDeploymentAllocationDecisions,
-      eligibleTransferDeployments,
-    }).tryMap(
-      ({
-        intermediateNetworkDeploymentAllocationDecisions,
-        eligibleTransferDeployments,
-      }) =>
-        mapValues(
-          this.multiNetworks.zip(
-            intermediateNetworkDeploymentAllocationDecisions,
-            eligibleTransferDeployments,
-          ),
-          ([allocationDecisions, eligibleTransferDeployments]: [
-            AllocationDecision[],
-            TransferredSubgraphDeployment[],
-          ]) => {
-            logger.debug(
-              `Found ${eligibleTransferDeployments.length} deployments eligible for transfer`,
-              { eligibleTransferDeployments },
-            )
-            const oneWeekAgo = Math.floor(Date.now() / 1_000) - 86_400 * 7
-            return allocationDecisions.map(decision => {
-              const matchingTransfer = eligibleTransferDeployments.find(
-                deployment =>
-                  deployment.ipfsHash == decision.deployment.ipfsHash &&
-                  Number(deployment.startedTransferToL2At) > oneWeekAgo,
-              )
-              if (matchingTransfer) {
-                logger.debug('Found a matching subgraph transfer', {
-                  matchingTransfer,
-                })
-                // L1 deployments being transferred need to be supported for one week post transfer
-                // to ensure continued support.
-                if (networkIsL1(matchingTransfer.protocolNetwork)) {
-                  decision.toAllocate = true
-                  decision.ruleMatch.activationCriteria =
-                    ActivationCriteria.L2_TRANSFER_SUPPORT
-                  logger.debug(
-                    `Allocating towards L1 subgraph deployment to support its transfer`,
-                    {
-                      subgraphDeployment: matchingTransfer,
-                      allocationDecision: decision,
-                    },
-                  )
-                }
-                // L2 Deployments
-                if (
-                  networkIsL2(matchingTransfer.protocolNetwork) &&
-                  !!matchingTransfer.transferredToL2
-                ) {
-                  decision.toAllocate = true
-                  decision.ruleMatch.activationCriteria =
-                    ActivationCriteria.L2_TRANSFER_SUPPORT
-                  logger.debug(
-                    `Allocating towards transferred L2 subgraph deployment`,
-                    {
-                      subgraphDeployment: matchingTransfer,
-                      allocationDecision: decision,
-                    },
-                  )
-                }
-              }
-              return decision
-            })
-          },
-        ),
-      {
-        onError: error =>
-          logger.warn(
-            `Failed to merge L2 transfer decisions, trying again later`,
-            {
-              error,
-            },
-          ),
-      },
-    )
-
     // let targetDeployments be an union of targetAllocations
     // and offchain subgraphs.
     const targetDeployments: Eventual<SubgraphDeploymentID[]> = join({

packages/indexer-agent/src/commands/common-options.ts

@@ -135,14 +135,6 @@ export function injectCommonStartupOptions(argv: Argv): Argv {
       required: true,
       group: 'Indexer Infrastructure',
     })
-    .option('enable-auto-migration-support', {
-      description:
-        'Auto migrate allocations from L1 to L2 (multi-network mode must be enabled)',
-      type: 'boolean',
-      required: false,
-      default: false,
-      group: 'Indexer Infrastructure',
-    })
     .option('deployment-management', {
       describe: 'Subgraph deployments management mode',
       required: false,

packages/indexer-agent/src/commands/start.ts

@@ -688,7 +688,6 @@ export async function run(
     indexerManagement: indexerManagementClient,
     networks,
     deploymentManagement: argv.deploymentManagement,
-    autoMigrationSupport: argv.enableAutoMigrationSupport,
     offchainSubgraphs: argv.offchainSubgraphs.map(
       (s: string) => new SubgraphDeploymentID(s),
     ),

packages/indexer-agent/src/types.ts

@@ -22,7 +22,6 @@ export interface AgentConfigs {
   indexerManagement: IndexerManagementClient
   networks: Network[]
   deploymentManagement: DeploymentManagementMode
-  autoMigrationSupport: boolean
   offchainSubgraphs: SubgraphDeploymentID[]
   pollingInterval: number
 }

packages/indexer-cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graphprotocol/indexer-cli",
-  "version": "0.25.2",
+  "version": "0.25.3",
   "description": "Indexer CLI for The Graph Network",
   "main": "./dist/cli.js",
   "files": [
@@ -27,7 +27,7 @@
   },
   "dependencies": {
     "@graphprotocol/common-ts": "3.0.1",
-    "@graphprotocol/indexer-common": "0.25.2",
+    "@graphprotocol/indexer-common": "0.25.3",
     "@iarna/toml": "2.2.5",
     "@thi.ng/iterators": "5.1.74",
     "@urql/core": "3.1.0",

packages/indexer-common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graphprotocol/indexer-common",
-  "version": "0.25.2",
+  "version": "0.25.3",
   "description": "Common library for Graph Protocol indexer components",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",

packages/indexer-common/src/allocations/graph-tally-collector.ts

@@ -236,20 +236,30 @@ export class GraphTallyCollector {
         })
         this.logger.trace(`[TAPv2] RAW DATA`, { ravs, allocations })
 
-        const pendingRAVsToProcess = ravs
-          .map((rav) => {
-            const signedRav = rav.getSignedRAV()
-            return {
+        // Create an object for O(1) allocation lookups instead of O(n) Array.find()
+        // This optimizes performance from O(n²) to O(n) for large datasets
+        const allocationMap: { [key: string]: Allocation } = {}
+        for (let i = 0; i < allocations.length; i++) {
+          const allocation = allocations[i]
+          allocationMap[allocation.id.toLowerCase()] = allocation
+        }
+
+        const pendingRAVsToProcess: RavWithAllocation[] = []
+        for (let i = 0; i < ravs.length; i++) {
+          const rav = ravs[i]
+          const signedRav = rav.getSignedRAV()
+          const allocationId = toAddress(
+            collectionIdToAllocationId(signedRav.rav.collectionId),
+          ).toLowerCase()
+          const allocation = allocationMap[allocationId] // O(1) lookup
+          if (allocation !== undefined) {
+            pendingRAVsToProcess.push({
               rav: signedRav,
-              allocation: allocations.find(
-                (a) =>
-                  a.id ===
-                  toAddress(collectionIdToAllocationId(signedRav.rav.collectionId)),
-              ),
+              allocation: allocation,
               payer: rav.payer,
-            }
-          })
-          .filter((rav) => rav.allocation !== undefined) as RavWithAllocation[] // this is safe because we filter out undefined allocations
+            })
+          }
+        }
         this.logger.trace(`[TAPv2] Pending RAVs to process`, {
           pendingRAVsToProcess: pendingRAVsToProcess.length,
         })

packages/indexer-common/src/allocations/tap-collector.ts

@@ -216,18 +216,32 @@ export class TapCollector {
           ravs: ravs.length,
           allocations: allocations.length,
         })
-        return ravs
-          .map((rav) => {
-            const signedRav = rav.getSignedRAV()
-            return {
+
+        // Create an object for O(1) allocation lookups instead of O(n) Array.find()
+        // This optimizes performance from O(n²) to O(n) for large datasets
+        const allocationMap: { [key: string]: Allocation } = {}
+        for (let i = 0; i < allocations.length; i++) {
+          const allocation = allocations[i]
+          allocationMap[allocation.id.toLowerCase()] = allocation
+        }
+
+        const results: RavWithAllocation[] = []
+        for (let i = 0; i < ravs.length; i++) {
+          const rav = ravs[i]
+          const signedRav = rav.getSignedRAV()
+          const allocationId = toAddress(
+            signedRav.rav.allocationId.toString(),
+          ).toLowerCase()
+          const allocation = allocationMap[allocationId] // O(1) lookup
+          if (allocation !== undefined) {
+            results.push({
               rav: signedRav,
-              allocation: allocations.find(
-                (a) => a.id === toAddress(signedRav.rav.allocationId.toString()),
-              ),
+              allocation: allocation,
               sender: rav.senderAddress,
-            }
-          })
-          .filter((rav) => rav.allocation !== undefined) as RavWithAllocation[] // this is safe because we filter out undefined allocations
+            })
+          }
+        }
+        return results
       },
       {
         onError: (err) =>