diff --git a/README.md b/README.md index b63b0b424..667d56a27 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,12 @@ Indexer Infrastructure --auto-allocation-min-batch-size Minimum number of allocation transactions inside a batch for AUTO management mode [number] [default: 1] + --auto-graft-resolver-limit Maximum depth of grafting dependency to + automatically + resolve [number] [default: 0] + --ipfs-endpoint Endpoint to an ipfs node to quickly + query subgraph manifest data` [string] + [default: "https://ipfs.network.thegraph.com"] Network Subgraph --network-subgraph-deployment Network subgraph deployment [string] diff --git a/docs/errors.md b/docs/errors.md index 4bd04fcc9..20814fa03 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -886,4 +886,13 @@ This is a sub-error of `IE069`. It is reported when the indexer agent doesn't ha **Solution** -Please provide a `epoch-subgraph-endpoint` and make sure graph node has consistent network configurations (`mainnet`, `goerli`, `gnosis`) and is on or after version 0.28.0. \ No newline at end of file +## IE072 + +**Summary** + +Please provide a `epoch-subgraph-endpoint` and make sure graph node has consistent network configurations (`mainnet`, `goerli`, `gnosis`) and is on or after version 0.28.0. +Failed to deploy subgraph deployment graft base. + +**Solution** + +Please make sure the auto graft depth resolver has correct limit, and that the graft base deployment has synced to the graft block before trying again - Set indexing rules for agent to periodically reconcile the deployment. diff --git a/packages/indexer-agent/src/__tests__/indexer.ts b/packages/indexer-agent/src/__tests__/indexer.ts index bb5a56496..3515df032 100644 --- a/packages/indexer-agent/src/__tests__/indexer.ts +++ b/packages/indexer-agent/src/__tests__/indexer.ts @@ -126,6 +126,7 @@ const setup = async () => { await sequelize.sync({ force: true }) const statusEndpoint = 'http://localhost:8030/graphql' + const ipfsEndpoint = 'https://ipfs.network.thegraph.com' const indexingStatusResolver = new IndexingStatusResolver({ logger: logger, statusEndpoint: 'statusEndpoint', @@ -139,6 +140,7 @@ const setup = async () => { }) const indexNodeIDs = ['node_1'] + const autoGraftResolverLimit = 1 indexerManagementClient = await createIndexerManagementClient({ models, address: toAddress(address), @@ -157,6 +159,8 @@ const setup = async () => { features: { injectDai: false, }, + ipfsEndpoint, + autoGraftResolverLimit, }) indexer = new Indexer( @@ -168,6 +172,8 @@ const setup = async () => { parseGRT('1000'), address, AllocationManagementMode.AUTO, + ipfsEndpoint, + autoGraftResolverLimit, ) } diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index 2e3272f65..b03ed3c32 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -702,7 +702,11 @@ class Agent { // Ensure the deployment is deployed to the indexer // Note: we're not waiting here, as sometimes indexing a subgraph // will block if the IPFS files cannot be retrieved - this.indexer.ensure(name, deployment) + try { + this.indexer.ensure(name, deployment) + } catch { + this.indexer.resolveGrafting(name, deployment, 0) + } }), ) diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index a368e04ac..57caf174d 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -257,6 +257,18 @@ export default { default: 100, group: 'Query Fees', }) + .option('ipfs-endpoint', { + description: `Endpoint to an ipfs node to quickly query subgraph manifest data`, + type: 'string', + default: 'https://ipfs.network.thegraph.com', + group: 'Indexer Infrastructure', + }) + .option('auto-graft-resolver-limit', { + description: `Maximum depth of grafting dependency to automatically resolve`, + type: 'number', + default: 0, + group: 'Indexer Infrastructure', + }) .option('inject-dai', { description: 'Inject the GRT to DAI/USDC conversion rate into cost model variables', @@ -370,6 +382,12 @@ export default { ) { return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.' } + if ( + !Number.isInteger(argv['auto-graft-resolver-limit']) || + argv['auto-graft-resolver-limit'] < 0 + ) { + return 'Invalid --auto-graft-resolver-limit provided. Must be >= 0 and an integer.' + } return true }) .option('collect-receipts-endpoint', { @@ -693,6 +711,8 @@ export default { networkMonitor, allocationManagementMode, autoAllocationMinBatchSize: argv.autoAllocationMinBatchSize, + ipfsEndpoint: argv.ipfsEndpoint, + autoGraftResolverLimit: argv.autoGraftResolverLimit, }) await createIndexerManagementServer({ @@ -711,6 +731,8 @@ export default { argv.defaultAllocationAmount, indexerAddress, allocationManagementMode, + argv.ipfsEndpoint, + argv.autoGraftResolverLimit, ) if (networkSubgraphDeploymentId !== undefined) { diff --git a/packages/indexer-agent/src/indexer.ts b/packages/indexer-agent/src/indexer.ts index e8ae4b6b3..8c7acaec0 100644 --- a/packages/indexer-agent/src/indexer.ts +++ b/packages/indexer-agent/src/indexer.ts @@ -31,6 +31,7 @@ import { } from '@graphprotocol/indexer-common' import { CombinedError } from '@urql/core' import pMap from 'p-map' +import yaml from 'yaml' const POI_DISPUTES_CONVERTERS_FROM_GRAPHQL: Record< keyof POIDisputeAttributes, @@ -84,6 +85,8 @@ export class Indexer { defaultAllocationAmount: BigNumber indexerAddress: string allocationManagementMode: AllocationManagementMode + ipfsEndpoint: string + autoGraftResolverLimit: number constructor( logger: Logger, @@ -94,12 +97,16 @@ export class Indexer { defaultAllocationAmount: BigNumber, indexerAddress: string, allocationManagementMode: AllocationManagementMode, + ipfsUrl: string, + autoGraftResolverLimit: number, ) { this.indexerManagement = indexerManagement this.statusResolver = statusResolver this.logger = logger this.indexerAddress = indexerAddress this.allocationManagementMode = allocationManagementMode + this.autoGraftResolverLimit = autoGraftResolverLimit + this.ipfsEndpoint = ipfsUrl + '/api/v0/cat?arg=' if (adminEndpoint.startsWith('https')) { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -735,6 +742,78 @@ export class Indexer { } } + // Simple fetch for subgraph manifest + async subgraphManifest(targetDeployment: SubgraphDeploymentID) { + const ipfsFile = await fetch( + this.ipfsEndpoint + targetDeployment.ipfsHash, + { + method: 'POST', + redirect: 'follow', + }, + ) + return yaml.parse(await ipfsFile.text()) + } + + // Recursive function for targetDeployment resolve grafting, add depth until reached to resolverDepth + async resolveGrafting( + name: string, + targetDeployment: SubgraphDeploymentID, + depth: number, + ): Promise { + // Matches "/depth-" followed by one or more digits + const depthRegex = /\/depth-\d+/ + const manifest = await this.subgraphManifest(targetDeployment) + + // No grafting dependency + if (!manifest.features || !manifest.features.includes('grafting')) { + // Ensure sync if at root of dependency + if (depth) { + await this.ensure(name, targetDeployment) + } + return + } + + // Default autoGraftResolverLimit is 0, essentially disabling auto-resolve + if (depth >= this.autoGraftResolverLimit) { + throw indexerError( + IndexerErrorCode.IE074, + `Grafting depth reached limit for auto resolve`, + ) + } + + try { + const baseDeployment = new SubgraphDeploymentID(manifest.graft.base) + let baseName = name.replace(depthRegex, `/depth-${depth}`) + if (baseName === name) { + // add depth suffix if didn't have one from targetDeployment + baseName += `/depth-${depth}` + } + await this.resolveGrafting(baseName, baseDeployment, depth + 1) + + // If base deployment has synced upto the graft block, then ensure the target deployment + // Otherwise just log to come back later + const graftStatus = await this.statusResolver.indexingStatus([ + baseDeployment, + ]) + // If base deployment synced to required block, try to sync the target and + // turn off syncing for the base deployment + if ( + graftStatus[0].chains[0].latestBlock && + graftStatus[0].chains[0].latestBlock.number >= manifest.graft.block + ) { + await this.ensure(name, targetDeployment) + } else { + this.logger.debug( + `Graft base deployment has yet to reach the graft block, try again later`, + ) + } + } catch { + throw indexerError( + IndexerErrorCode.IE074, + `Base deployment hasn't synced to the graft block, try again later`, + ) + } + } async deploy( name: string, deployment: SubgraphDeploymentID, @@ -751,7 +830,7 @@ export class Indexer { node_id: node_id, }) if (response.error) { - throw response.error + throw indexerError(IndexerErrorCode.IE026, response.error) } this.logger.info(`Successfully deployed subgraph deployment`, { name, diff --git a/packages/indexer-common/src/errors.ts b/packages/indexer-common/src/errors.ts index a5433f612..0f61aecef 100644 --- a/packages/indexer-common/src/errors.ts +++ b/packages/indexer-common/src/errors.ts @@ -84,6 +84,7 @@ export enum IndexerErrorCode { IE071 = 'IE071', IE072 = 'IE072', IE073 = 'IE073', + IE074 = 'IE074', } export const INDEXER_ERROR_MESSAGES: Record = { @@ -161,6 +162,7 @@ export const INDEXER_ERROR_MESSAGES: Record = { IE071: 'Add Epoch subgraph support for non-protocol chains', IE072: 'Failed to execute batch tx (contract: staking)', IE073: 'Failed to query subgraph features from indexing statuses endpoint', + IE074: 'Failed to deploy the graft base for the target deployment', } export type IndexerErrorCause = unknown diff --git a/packages/indexer-common/src/indexer-management/__tests__/resolvers/actions.ts b/packages/indexer-common/src/indexer-management/__tests__/resolvers/actions.ts index c7095f6d6..6b436ab32 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/resolvers/actions.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/resolvers/actions.ts @@ -359,7 +359,11 @@ const setup = async () => { injectDai: true, }, }) - mockedSubgraphManager = new SubgraphManager('fake endpoint', ['fake node id']) + mockedSubgraphManager = new SubgraphManager( + 'fake endpoint', + ['fake node id'], + indexingStatusResolver, + ) allocationManager = new AllocationManager( contracts, logger, diff --git a/packages/indexer-common/src/indexer-management/allocations.ts b/packages/indexer-common/src/indexer-management/allocations.ts index 59b851475..89de40a7e 100644 --- a/packages/indexer-common/src/indexer-management/allocations.ts +++ b/packages/indexer-common/src/indexer-management/allocations.ts @@ -336,6 +336,14 @@ export class AllocationManager { deployment.ipfsHash, ) + // Ensure graft dependency is resolved + await this.subgraphManager.resolveGrafting( + logger, + this.models, + deployment, + indexNode, + 0, + ) // Ensure subgraph is deployed before allocating await this.subgraphManager.ensure( logger, diff --git a/packages/indexer-common/src/indexer-management/client.ts b/packages/indexer-common/src/indexer-management/client.ts index df375449a..0785597a1 100644 --- a/packages/indexer-common/src/indexer-management/client.ts +++ b/packages/indexer-common/src/indexer-management/client.ts @@ -457,6 +457,8 @@ export interface IndexerManagementClientOptions { networkMonitor?: NetworkMonitor allocationManagementMode?: AllocationManagementMode autoAllocationMinBatchSize?: number + ipfsEndpoint?: string + autoGraftResolverLimit?: number } export class IndexerManagementClient extends Client { @@ -522,6 +524,8 @@ export const createIndexerManagementClient = async ( networkMonitor, allocationManagementMode, autoAllocationMinBatchSize, + ipfsEndpoint, + autoGraftResolverLimit, } = options const schema = buildSchema(print(SCHEMA_SDL)) const resolvers = { @@ -535,7 +539,13 @@ export const createIndexerManagementClient = async ( const dai: WritableEventual = mutable() - const subgraphManager = new SubgraphManager(deploymentManagementEndpoint, indexNodeIDs) + const subgraphManager = new SubgraphManager( + deploymentManagementEndpoint, + indexNodeIDs, + indexingStatusResolver, + ipfsEndpoint, + autoGraftResolverLimit, + ) let allocationManager: AllocationManager | undefined = undefined let actionManager: ActionManager | undefined = undefined diff --git a/packages/indexer-common/src/indexer-management/monitor.ts b/packages/indexer-common/src/indexer-management/monitor.ts index 4e77864e5..c291cc916 100644 --- a/packages/indexer-common/src/indexer-management/monitor.ts +++ b/packages/indexer-common/src/indexer-management/monitor.ts @@ -630,9 +630,9 @@ export class NetworkMonitor { if (!result.data.network) { // If the network is missing, it means it is not registered in the Epoch Subgraph. - throw new Error( - `Failed to query EBO for ${networkID}'s latest valid epoch number: Network not found`, - ) + const err = `Failed to query EBO for ${networkID}'s latest valid epoch number: Network not found` + this.logger.error(err) + throw indexerError(IndexerErrorCode.IE071, err) } if (!result.data.network.latestValidBlockNumber) { diff --git a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts index 35797846b..6f2fa9891 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts @@ -483,6 +483,14 @@ export default { subgraphDeploymentID.ipfsHash, ) + // Ensure grafting dependencies are resolved + await subgraphManager.resolveGrafting( + logger, + models, + subgraphDeploymentID, + indexNode, + 0, + ) // Ensure subgraph is deployed before allocating await subgraphManager.ensure( logger, diff --git a/packages/indexer-common/src/indexer-management/subgraphs.ts b/packages/indexer-common/src/indexer-management/subgraphs.ts index 7d90386f8..8e6625248 100644 --- a/packages/indexer-common/src/indexer-management/subgraphs.ts +++ b/packages/indexer-common/src/indexer-management/subgraphs.ts @@ -2,16 +2,34 @@ import { indexerError, IndexerErrorCode, IndexerManagementModels, + IndexingDecisionBasis, + IndexingRuleAttributes, + SubgraphIdentifierType, + upsertIndexingRule, + fetchIndexingRules, + INDEXING_RULE_GLOBAL, + IndexingStatusResolver, } from '@graphprotocol/indexer-common' import { Logger, SubgraphDeploymentID } from '@graphprotocol/common-ts' import jayson, { Client as RpcClient } from 'jayson/promise' import pTimeout from 'p-timeout' +import fetch from 'isomorphic-fetch' +import yaml from 'yaml' export class SubgraphManager { client: RpcClient indexNodeIDs: string[] + statusResolver: IndexingStatusResolver + autoGraftResolverLimit: number + ipfsEndpoint?: string - constructor(endpoint: string, indexNodeIDs: string[]) { + constructor( + endpoint: string, + indexNodeIDs: string[], + statusResolver: IndexingStatusResolver, + ipfsUrl?: string, + autoGraftResolverLimit?: number, + ) { if (endpoint.startsWith('https')) { // eslint-disable-next-line @typescript-eslint/no-explicit-any this.client = jayson.Client.https(endpoint as any) @@ -20,6 +38,9 @@ export class SubgraphManager { this.client = jayson.Client.http(endpoint as any) } this.indexNodeIDs = indexNodeIDs + this.statusResolver = statusResolver + this.ipfsEndpoint = ipfsUrl + '/api/v0/cat?arg=' + this.autoGraftResolverLimit = autoGraftResolverLimit ?? 0 } async createSubgraph(logger: Logger, name: string): Promise { @@ -76,7 +97,7 @@ export class SubgraphManager { const response = await pTimeout(requestPromise, 120000) if (response.error) { - throw response.error + throw indexerError(IndexerErrorCode.IE026, response.error) } logger.info(`Successfully deployed subgraph`, { name, @@ -84,8 +105,18 @@ export class SubgraphManager { endpoints: response.result, }) - // TODO: Insert an offchain indexing rule if one matching this deployment doesn't yet exist // Will be useful for supporting deploySubgraph resolver + const indexingRules = (await fetchIndexingRules(models, false)) + .filter((rule) => rule.identifier != INDEXING_RULE_GLOBAL) + .map((rule) => new SubgraphDeploymentID(rule.identifier)) + if (!indexingRules.includes(deployment)) { + const offchainIndexingRule = { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.OFFCHAIN, + } as Partial + await upsertIndexingRule(logger, models, offchainIndexingRule) + } } catch (error) { const err = indexerError(IndexerErrorCode.IE026, error) logger.error(`Failed to deploy subgraph deployment`, { @@ -212,4 +243,92 @@ export class SubgraphManager { throw error } } + + // Simple fetch for subgraph manifest + async subgraphManifest(targetDeployment: SubgraphDeploymentID) { + const ipfsFile = await fetch(this.ipfsEndpoint + targetDeployment.ipfsHash, { + method: 'POST', + redirect: 'follow', + }) + return yaml.parse(await ipfsFile.text()) + } + + // Recursive function for targetDeployment resolve grafting, add depth until reached to resolverDepth + async resolveGrafting( + logger: Logger, + models: IndexerManagementModels, + targetDeployment: SubgraphDeploymentID, + indexNode: string | undefined, + depth: number, + ): Promise { + const manifest = await this.subgraphManifest(targetDeployment) + const name = `indexer-agent/${targetDeployment.ipfsHash.slice(-10)}` + + // No grafting or at root of dependency + if (!manifest.features || !manifest.features.includes('grafting')) { + if (depth) { + await this.ensure(logger, models, name, targetDeployment, indexNode) + } + return + } + // Default limit set to 0, disable auto-resolve of grafting dependencies + if (depth >= this.autoGraftResolverLimit) { + throw indexerError( + IndexerErrorCode.IE074, + `Grafting depth reached limit for auto resolve`, + ) + } + + try { + const baseDeployment = new SubgraphDeploymentID(manifest.graft.base) + let baseName = name.replace(this.depthRegex, `/depth-${depth}`) + if (baseName === name) { + // add depth suffix if didn't have one from targetDeployment + baseName += `/depth-${depth}` + } + await this.resolveGrafting(logger, models, baseDeployment, indexNode, depth + 1) + + // If base deployment has synced upto the graft block, then ensure the target deployment + // Otherwise just log to come back later + const graftStatus = await this.statusResolver.indexingStatus([baseDeployment]) + // If base deployment synced to required block, try to sync the target and + // turn off syncing for the base deployment + if ( + graftStatus[0].chains[0].latestBlock && + graftStatus[0].chains[0].latestBlock.number >= manifest.graft.block + ) { + await this.ensure(logger, models, name, targetDeployment, indexNode) + // At this point, can safely set NEVER to graft base deployment + await this.stop_sync(logger, baseDeployment, models) + } else { + logger.debug( + `Graft base deployment has yet to reach the graft block, try again later`, + ) + } + } catch { + throw indexerError( + IndexerErrorCode.IE074, + `Base deployment hasn't synced to the graft block, try again later`, + ) + } + } + + /** + * Matches "/depth-" followed by one or more digits + */ + depthRegex = /\/depth-\d+/ + + async stop_sync( + logger: Logger, + deployment: SubgraphDeploymentID, + models: IndexerManagementModels, + ) { + const neverIndexingRule = { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.NEVER, + } as Partial + + await upsertIndexingRule(logger, models, neverIndexingRule) + } }