diff --git a/.gitignore b/.gitignore index 7f0569a3..5a76411b 100644 --- a/.gitignore +++ b/.gitignore @@ -82,8 +82,6 @@ apps/indexer-coordinator/dist/* .data .projects .eslintcache -<<<<<<< HEAD -======= +.ipfs .env ->>>>>>> main diff --git a/apps/indexer-coordinator/dev/docker-compose.yml b/apps/indexer-coordinator/dev/docker-compose.yml index d58e1a80..f5803c2e 100644 --- a/apps/indexer-coordinator/dev/docker-compose.yml +++ b/apps/indexer-coordinator/dev/docker-compose.yml @@ -30,20 +30,22 @@ services: container_name: indexer_coordinator image: coordinator:latest build: - context: ../ + context: ../../../ dockerfile: Dockerfile ports: - 8000:8000 depends_on: "postgres": condition: service_healthy + "ipfs": + condition: service_healthy restart: always volumes: - .projects:/usr/projects - /var/run/docker.sock:/var/run/docker.sock command: - --network=testnet - - --ws-endpoint="https://polygon-mumbai.infura.io/v3/4458cf4d1689497b9a38b1d6bbf05e78 + - --ws-endpoint=https://polygon-mumbai.infura.io/v3/4458cf4d1689497b9a38b1d6bbf05e78 - --postgres-host=postgres - --postgres-password=pos_z8X # use the same password in the db config @@ -51,6 +53,8 @@ services: - --use-prerelease - --port=8000 - --debug + - --ipfs=http://indexer_ipfs:8080/api/v0/ + - --mmrPath=/home healthcheck: test: [ @@ -93,6 +97,20 @@ services: timeout: 5s retries: 3 + ipfs: + image: ipfs/kubo:master-2023-07-20-1fe17a4 + container_name: indexer_ipfs + volumes: + - .ipfs/export:/export + - .ipfs/data:/data/ipfs + - ./ipfs.sh:/container-init.d/ipfs.sh + ports: + - 4001:4001 + - 4001:4001/udp + - 127.0.0.1:5001:5001 + expose: + - 8080 + networks: default: name: indexer_services \ No newline at end of file diff --git a/apps/indexer-coordinator/dev/ipfs.sh b/apps/indexer-coordinator/dev/ipfs.sh new file mode 100644 index 00000000..c1259fd5 --- /dev/null +++ b/apps/indexer-coordinator/dev/ipfs.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -ex + +ipfs_bootstrap=$(ipfs bootstrap list) + +if echo "$ipfs_bootstrap" | grep -q "subquery.network"; then + echo "There are already one or more subquery.network bootstrap nodes, skipping..." +else + ipfs bootstrap add /dns4/ipfs-swarm.subquery.network/tcp/19988/p2p/12D3KooWHEEjciF2JmDukCkWW93tQ7eJYs16PWqEo81GrXz82DUL + ipfs bootstrap add /dns4/ipfs-swarm-a-lh.subquery.network/tcp/19988/p2p/12D3KooWForH2nsSRN5cynPhoona6re1nw2EcimQJxHnicd1yqUV + ipfs bootstrap add /dns4/ipfs-swarm-b-lh.subquery.network/tcp/19988/p2p/12D3KooWPhsrviSKFTKawpW3bRAdLZ89jhXdYuszAys4YwL3RMn3 +fi diff --git a/apps/indexer-coordinator/src/project/project.service.ts b/apps/indexer-coordinator/src/project/project.service.ts index b88cc4c8..a1134a9c 100644 --- a/apps/indexer-coordinator/src/project/project.service.ts +++ b/apps/indexer-coordinator/src/project/project.service.ts @@ -4,6 +4,7 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { GraphqlQueryClient, IPFS_URLS, IPFSClient, NETWORK_CONFIGS } from '@subql/network-clients'; +import { argv } from 'src/yargs'; import { Not, Repository } from 'typeorm'; import { Config } from '../configure/configure.module'; @@ -27,7 +28,7 @@ import { schemaName, } from '../utils/docker'; import { debugLogger, getLogger } from '../utils/logger'; -import { nodeConfigs, projectConfigChanged } from '../utils/project'; +import { nodeConfigs, projectConfigChanged, IPFS_URL } from '../utils/project'; import { GET_DEPLOYMENT, GET_INDEXER_PROJECTS } from '../utils/queries'; import { ProjectEvent } from '../utils/subscription'; import { PortService } from './port.service'; @@ -212,6 +213,8 @@ export class ProjectService { const postgres = this.config.postgres; const dockerNetwork = this.config.dockerNetwork; + const mmrPath = argv['mmrPath'].replace(/\/$/, ''); + const item: TemplateType = { deploymentID: project.id, dbSchema: schemaName(project.id), @@ -220,6 +223,8 @@ export class ProjectService { postgres, mmrStoreType, dockerNetwork, + ipfsUrl: IPFS_URL, + mmrPath, ...baseConfig, ...advancedConfig, }; diff --git a/apps/indexer-coordinator/src/project/types.ts b/apps/indexer-coordinator/src/project/types.ts index ae4b2249..38a91673 100644 --- a/apps/indexer-coordinator/src/project/types.ts +++ b/apps/indexer-coordinator/src/project/types.ts @@ -21,6 +21,8 @@ export type TemplateType = { dbSchema: string; postgres: Postgres; dockerNetwork: string; + ipfsUrl: string; + mmrPath: string; worker: number; batchSize: number; timeout: number; diff --git a/apps/indexer-coordinator/src/utils/project.ts b/apps/indexer-coordinator/src/utils/project.ts index 3a00d638..4fb52693 100644 --- a/apps/indexer-coordinator/src/utils/project.ts +++ b/apps/indexer-coordinator/src/utils/project.ts @@ -4,8 +4,8 @@ import { IPFSClient } from '@subql/network-clients'; import yaml from 'js-yaml'; import { isEqual } from 'lodash'; - import { Project, ProjectAdvancedConfig, ProjectBaseConfig } from '../project/project.model'; +import { argv } from '../yargs'; // manifest types export type Runner = { @@ -37,7 +37,7 @@ export type PartialIpfsDeploymentManifest = { export type ChainType = 'near' | 'flare' | 'cosmos' | 'algorand' | 'substrate' | 'ethereum'; -export const IPFS_URL = 'https://authipfs.subquery.network/ipfs/api/v0'; +export const IPFS_URL = argv['ipfs'] ?? 'https://authipfs.subquery.network/ipfs/api/v0'; const clientSDK = new IPFSClient(IPFS_URL); export function projectConfigChanged( diff --git a/apps/indexer-coordinator/src/utils/template.yml b/apps/indexer-coordinator/src/utils/template.yml index cd1ed067..1e97e152 100644 --- a/apps/indexer-coordinator/src/utils/template.yml +++ b/apps/indexer-coordinator/src/utils/template.yml @@ -19,7 +19,7 @@ services: command: - -f=ipfs://{{deploymentID}} - -d={{networkDictionary}} - - --ipfs=https://authipfs.subquery.network/ipfs/api/v0 + - --ipfs={{ipfsUrl}} - --network-endpoint={{{networkEndpoint}}} - --db-schema={{dbSchema}} - --port={{servicePort}} diff --git a/apps/indexer-coordinator/src/yargs.ts b/apps/indexer-coordinator/src/yargs.ts index ec229dcf..1839c592 100644 --- a/apps/indexer-coordinator/src/yargs.ts +++ b/apps/indexer-coordinator/src/yargs.ts @@ -82,7 +82,7 @@ function getYargsOption() { mmrPath: { type: 'string', describe: 'The local path to store the mmr data', - default: '/home/indexer-service', + default: '/home', group: Groups.node, }, 'docker-network': {