indexer-common,indexer-agent: auto graft resolve with limit

README.md

@@ -190,6 +190,12 @@ Indexer Infrastructure
   --auto-allocation-min-batch-size                 Minimum number of allocation 
                                         transactions inside a batch for AUTO 
                                         management mode    [number] [default: 1]
+  --auto-graft-resolver-limit           Maximum depth of grafting dependency to 
+                                        automatically 
+                                        resolve            [number] [default: 0]
+  --ipfs-endpoint                       Endpoint to an ipfs node to quickly 
+                                        query subgraph manifest data`   [string]
+                                        [default: "https://ipfs.network.thegraph.com"]
 
 Network Subgraph
   --network-subgraph-deployment   Network subgraph deployment           [string]

docs/errors.md

@@ -886,4 +886,13 @@ This is a sub-error of `IE069`. It is reported when the indexer agent doesn't ha
 
 **Solution**
 
-Please provide a `epoch-subgraph-endpoint` and make sure graph node has consistent network configurations (`mainnet`, `goerli`, `gnosis`) and is on or after version 0.28.0.
+## IE072
+
+**Summary**
+
+Please provide a `epoch-subgraph-endpoint` and make sure graph node has consistent network configurations (`mainnet`, `goerli`, `gnosis`) and is on or after version 0.28.0.
+Failed to deploy subgraph deployment graft base.
+
+**Solution**
+
+Please make sure the auto graft depth resolver has correct limit, and that the graft base deployment has synced to the graft block before trying again - Set indexing rules for agent to periodically reconcile the deployment.

packages/indexer-agent/src/__tests__/indexer.ts

@@ -126,6 +126,7 @@ const setup = async () => {
   await sequelize.sync({ force: true })
 
   const statusEndpoint = 'http://localhost:8030/graphql'
+  const ipfsEndpoint = 'https://ipfs.network.thegraph.com'
   const indexingStatusResolver = new IndexingStatusResolver({
     logger: logger,
     statusEndpoint: 'statusEndpoint',
@@ -139,6 +140,7 @@ const setup = async () => {
   })
 
   const indexNodeIDs = ['node_1']
+  const autoGraftResolverLimit = 1
   indexerManagementClient = await createIndexerManagementClient({
     models,
     address: toAddress(address),
@@ -157,6 +159,8 @@ const setup = async () => {
     features: {
       injectDai: false,
     },
+    ipfsEndpoint,
+    autoGraftResolverLimit,
   })
 
   indexer = new Indexer(
@@ -168,6 +172,8 @@ const setup = async () => {
     parseGRT('1000'),
     address,
     AllocationManagementMode.AUTO,
+    ipfsEndpoint,
+    autoGraftResolverLimit,
   )
 }
 

packages/indexer-agent/src/agent.ts

@@ -702,7 +702,11 @@ class Agent {
         // Ensure the deployment is deployed to the indexer
         // Note: we're not waiting here, as sometimes indexing a subgraph
         // will block if the IPFS files cannot be retrieved
-        this.indexer.ensure(name, deployment)
+        try {
+          this.indexer.ensure(name, deployment)
+        } catch {
+          this.indexer.resolveGrafting(name, deployment, 0)
+        }
       }),
     )
 

packages/indexer-agent/src/commands/start.ts

@@ -257,6 +257,18 @@ export default {
         default: 100,
         group: 'Query Fees',
       })
+      .option('ipfs-endpoint', {
+        description: `Endpoint to an ipfs node to quickly query subgraph manifest data`,
+        type: 'string',
+        default: 'https://ipfs.network.thegraph.com',
+        group: 'Indexer Infrastructure',
+      })
+      .option('auto-graft-resolver-limit', {
+        description: `Maximum depth of grafting dependency to automatically resolve`,
+        type: 'number',
+        default: 0,
+        group: 'Indexer Infrastructure',
+      })
       .option('inject-dai', {
         description:
           'Inject the GRT to DAI/USDC conversion rate into cost model variables',
@@ -370,6 +382,12 @@ export default {
         ) {
           return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.'
         }
+        if (
+          !Number.isInteger(argv['auto-graft-resolver-limit']) ||
+          argv['auto-graft-resolver-limit'] < 0
+        ) {
+          return 'Invalid --auto-graft-resolver-limit provided. Must be >= 0 and an integer.'
+        }
         return true
       })
       .option('collect-receipts-endpoint', {
@@ -693,6 +711,8 @@ export default {
       networkMonitor,
       allocationManagementMode,
       autoAllocationMinBatchSize: argv.autoAllocationMinBatchSize,
+      ipfsEndpoint: argv.ipfsEndpoint,
+      autoGraftResolverLimit: argv.autoGraftResolverLimit,
     })
 
     await createIndexerManagementServer({
@@ -711,6 +731,8 @@ export default {
       argv.defaultAllocationAmount,
       indexerAddress,
       allocationManagementMode,
+      argv.ipfsEndpoint,
+      argv.autoGraftResolverLimit,
     )
 
     if (networkSubgraphDeploymentId !== undefined) {

packages/indexer-agent/src/indexer.ts

@@ -31,6 +31,7 @@ import {
 } from '@graphprotocol/indexer-common'
 import { CombinedError } from '@urql/core'
 import pMap from 'p-map'
+import yaml from 'yaml'
 
 const POI_DISPUTES_CONVERTERS_FROM_GRAPHQL: Record<
   keyof POIDisputeAttributes,
@@ -84,6 +85,8 @@ export class Indexer {
   defaultAllocationAmount: BigNumber
   indexerAddress: string
   allocationManagementMode: AllocationManagementMode
+  ipfsEndpoint: string
+  autoGraftResolverLimit: number
 
   constructor(
     logger: Logger,
@@ -94,12 +97,16 @@ export class Indexer {
     defaultAllocationAmount: BigNumber,
     indexerAddress: string,
     allocationManagementMode: AllocationManagementMode,
+    ipfsUrl: string,
+    autoGraftResolverLimit: number,
   ) {
     this.indexerManagement = indexerManagement
     this.statusResolver = statusResolver
     this.logger = logger
     this.indexerAddress = indexerAddress
     this.allocationManagementMode = allocationManagementMode
+    this.autoGraftResolverLimit = autoGraftResolverLimit
+    this.ipfsEndpoint = ipfsUrl + '/api/v0/cat?arg='
 
     if (adminEndpoint.startsWith('https')) {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -735,6 +742,78 @@ export class Indexer {
     }
   }
 
+  // Simple fetch for subgraph manifest
+  async subgraphManifest(targetDeployment: SubgraphDeploymentID) {
+    const ipfsFile = await fetch(
+      this.ipfsEndpoint + targetDeployment.ipfsHash,
+      {
+        method: 'POST',
+        redirect: 'follow',
+      },
+    )
+    return yaml.parse(await ipfsFile.text())
+  }
+
+  // Recursive function for targetDeployment resolve grafting, add depth until reached to resolverDepth
+  async resolveGrafting(
+    name: string,
+    targetDeployment: SubgraphDeploymentID,
+    depth: number,
+  ): Promise<void> {
+    // Matches "/depth-" followed by one or more digits
+    const depthRegex = /\/depth-\d+/
+    const manifest = await this.subgraphManifest(targetDeployment)
+
+    // No grafting dependency
+    if (!manifest.features || !manifest.features.includes('grafting')) {
+      // Ensure sync if at root of dependency
+      if (depth) {
+        await this.ensure(name, targetDeployment)
+      }
+      return
+    }
+
+    // Default autoGraftResolverLimit is 0, essentially disabling auto-resolve
+    if (depth >= this.autoGraftResolverLimit) {
+      throw indexerError(
+        IndexerErrorCode.IE074,
+        `Grafting depth reached limit for auto resolve`,
+      )
+    }
+
+    try {
+      const baseDeployment = new SubgraphDeploymentID(manifest.graft.base)
+      let baseName = name.replace(depthRegex, `/depth-${depth}`)
+      if (baseName === name) {
+        // add depth suffix if didn't have one from targetDeployment
+        baseName += `/depth-${depth}`
+      }
+      await this.resolveGrafting(baseName, baseDeployment, depth + 1)
+
+      // If base deployment has synced upto the graft block, then ensure the target deployment
+      // Otherwise just log to come back later
+      const graftStatus = await this.statusResolver.indexingStatus([
+        baseDeployment,
+      ])
+      // If base deployment synced to required block, try to sync the target and
+      // turn off syncing for the base deployment
+      if (
+        graftStatus[0].chains[0].latestBlock &&
+        graftStatus[0].chains[0].latestBlock.number >= manifest.graft.block
+      ) {
+        await this.ensure(name, targetDeployment)
+      } else {
+        this.logger.debug(
+          `Graft base deployment has yet to reach the graft block, try again later`,
+        )
+      }
+    } catch {
+      throw indexerError(
+        IndexerErrorCode.IE074,
+        `Base deployment hasn't synced to the graft block, try again later`,
+      )
+    }
+  }
   async deploy(
     name: string,
     deployment: SubgraphDeploymentID,
@@ -751,7 +830,7 @@ export class Indexer {
         node_id: node_id,
       })
       if (response.error) {
-        throw response.error
+        throw indexerError(IndexerErrorCode.IE026, response.error)
       }
       this.logger.info(`Successfully deployed subgraph deployment`, {
         name,

packages/indexer-common/src/errors.ts

@@ -84,6 +84,7 @@ export enum IndexerErrorCode {
   IE071 = 'IE071',
   IE072 = 'IE072',
   IE073 = 'IE073',
+  IE074 = 'IE074',
 }
 
 export const INDEXER_ERROR_MESSAGES: Record<IndexerErrorCode, string> = {
@@ -161,6 +162,7 @@ export const INDEXER_ERROR_MESSAGES: Record<IndexerErrorCode, string> = {
   IE071: 'Add Epoch subgraph support for non-protocol chains',
   IE072: 'Failed to execute batch tx (contract: staking)',
   IE073: 'Failed to query subgraph features from indexing statuses endpoint',
+  IE074: 'Failed to deploy the graft base for the target deployment',
 }
 
 export type IndexerErrorCause = unknown

packages/indexer-common/src/indexer-management/__tests__/resolvers/actions.ts

@@ -359,7 +359,11 @@ const setup = async () => {
       injectDai: true,
     },
   })
-  mockedSubgraphManager = new SubgraphManager('fake endpoint', ['fake node id'])
+  mockedSubgraphManager = new SubgraphManager(
+    'fake endpoint',
+    ['fake node id'],
+    indexingStatusResolver,
+  )
   allocationManager = new AllocationManager(
     contracts,
     logger,

packages/indexer-common/src/indexer-management/allocations.ts

@@ -336,6 +336,14 @@ export class AllocationManager {
       deployment.ipfsHash,
     )
 
+    // Ensure graft dependency is resolved
+    await this.subgraphManager.resolveGrafting(
+      logger,
+      this.models,
+      deployment,
+      indexNode,
+      0,
+    )
     // Ensure subgraph is deployed before allocating
     await this.subgraphManager.ensure(
       logger,

packages/indexer-common/src/indexer-management/client.ts

@@ -457,6 +457,8 @@ export interface IndexerManagementClientOptions {
   networkMonitor?: NetworkMonitor
   allocationManagementMode?: AllocationManagementMode
   autoAllocationMinBatchSize?: number
+  ipfsEndpoint?: string
+  autoGraftResolverLimit?: number
 }
 
 export class IndexerManagementClient extends Client {
@@ -522,6 +524,8 @@ export const createIndexerManagementClient = async (
     networkMonitor,
     allocationManagementMode,
     autoAllocationMinBatchSize,
+    ipfsEndpoint,
+    autoGraftResolverLimit,
   } = options
   const schema = buildSchema(print(SCHEMA_SDL))
   const resolvers = {
@@ -535,7 +539,13 @@ export const createIndexerManagementClient = async (
 
   const dai: WritableEventual<string> = mutable()
 
-  const subgraphManager = new SubgraphManager(deploymentManagementEndpoint, indexNodeIDs)
+  const subgraphManager = new SubgraphManager(
+    deploymentManagementEndpoint,
+    indexNodeIDs,
+    indexingStatusResolver,
+    ipfsEndpoint,
+    autoGraftResolverLimit,
+  )
   let allocationManager: AllocationManager | undefined = undefined
   let actionManager: ActionManager | undefined = undefined
 

packages/indexer-common/src/indexer-management/monitor.ts

@@ -630,9 +630,9 @@ export class NetworkMonitor {
 
       if (!result.data.network) {
         // If the network is missing, it means it is not registered in the Epoch Subgraph.
-        throw new Error(
-          `Failed to query EBO for ${networkID}'s latest valid epoch number: Network not found`,
-        )
+        const err = `Failed to query EBO for ${networkID}'s latest valid epoch number: Network not found`
+        this.logger.error(err)
+        throw indexerError(IndexerErrorCode.IE071, err)
       }
 
       if (!result.data.network.latestValidBlockNumber) {

packages/indexer-common/src/indexer-management/resolvers/allocations.ts

@@ -483,6 +483,14 @@ export default {
         subgraphDeploymentID.ipfsHash,
       )
 
+      // Ensure grafting dependencies are resolved
+      await subgraphManager.resolveGrafting(
+        logger,
+        models,
+        subgraphDeploymentID,
+        indexNode,
+        0,
+      )
       // Ensure subgraph is deployed before allocating
       await subgraphManager.ensure(
         logger,