support docker/local ipfs

.gitignore

@@ -82,8 +82,6 @@ apps/indexer-coordinator/dist/*
 .data
 .projects
 .eslintcache
-<<<<<<< HEAD
-=======
+.ipfs
 
 .env
->>>>>>> main

apps/indexer-coordinator/dev/docker-compose.yml

@@ -30,27 +30,31 @@ services:
     container_name: indexer_coordinator
     image: coordinator:latest
     build:
-      context: ../
+      context: ../../../
       dockerfile: Dockerfile
     ports:
       - 8000:8000
     depends_on:
       "postgres":
         condition: service_healthy
+      "ipfs":
+        condition: service_healthy
     restart: always
     volumes:
       - .projects:/usr/projects
       - /var/run/docker.sock:/var/run/docker.sock
     command:
       - --network=testnet
-      - --ws-endpoint="https://polygon-mumbai.infura.io/v3/4458cf4d1689497b9a38b1d6bbf05e78
+      - --ws-endpoint=https://polygon-mumbai.infura.io/v3/4458cf4d1689497b9a38b1d6bbf05e78
       - --postgres-host=postgres
       - --postgres-password=pos_z8X
       # use the same password in the db config
       - --postgres-port=5432
       - --use-prerelease
       - --port=8000
       - --debug
+      - --ipfs=http://indexer_ipfs:8080/api/v0/
+      - --mmrPath=/home
     healthcheck:
       test:
         [
@@ -93,6 +97,20 @@ services:
       timeout: 5s
       retries: 3
 
+  ipfs:
+    image: ipfs/kubo:master-2023-07-20-1fe17a4
+    container_name: indexer_ipfs
+    volumes:
+      - .ipfs/export:/export
+      - .ipfs/data:/data/ipfs
+      - ./ipfs.sh:/container-init.d/ipfs.sh
+    ports:
+      - 4001:4001
+      - 4001:4001/udp
+      - 127.0.0.1:5001:5001
+    expose:
+      - 8080
+
 networks:
   default:
     name: indexer_services

apps/indexer-coordinator/dev/ipfs.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -ex
+
+ipfs_bootstrap=$(ipfs bootstrap list)
+
+if echo "$ipfs_bootstrap" | grep -q "subquery.network"; then
+  echo "There are already one or more subquery.network bootstrap nodes, skipping..."
+else
+  ipfs bootstrap add /dns4/ipfs-swarm.subquery.network/tcp/19988/p2p/12D3KooWHEEjciF2JmDukCkWW93tQ7eJYs16PWqEo81GrXz82DUL
+  ipfs bootstrap add /dns4/ipfs-swarm-a-lh.subquery.network/tcp/19988/p2p/12D3KooWForH2nsSRN5cynPhoona6re1nw2EcimQJxHnicd1yqUV
+  ipfs bootstrap add /dns4/ipfs-swarm-b-lh.subquery.network/tcp/19988/p2p/12D3KooWPhsrviSKFTKawpW3bRAdLZ89jhXdYuszAys4YwL3RMn3
+fi

apps/indexer-coordinator/src/project/project.service.ts

@@ -4,6 +4,7 @@
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { GraphqlQueryClient, IPFS_URLS, IPFSClient, NETWORK_CONFIGS } from '@subql/network-clients';
+import { argv } from 'src/yargs';
 import { Not, Repository } from 'typeorm';
 
 import { Config } from '../configure/configure.module';
@@ -27,7 +28,7 @@ import {
   schemaName,
 } from '../utils/docker';
 import { debugLogger, getLogger } from '../utils/logger';
-import { nodeConfigs, projectConfigChanged } from '../utils/project';
+import { nodeConfigs, projectConfigChanged, IPFS_URL } from '../utils/project';
 import { GET_DEPLOYMENT, GET_INDEXER_PROJECTS } from '../utils/queries';
 import { ProjectEvent } from '../utils/subscription';
 import { PortService } from './port.service';
@@ -212,6 +213,8 @@ export class ProjectService {
     const postgres = this.config.postgres;
     const dockerNetwork = this.config.dockerNetwork;
 
+    const mmrPath = argv['mmrPath'].replace(/\/$/, '');
+
     const item: TemplateType = {
       deploymentID: project.id,
       dbSchema: schemaName(project.id),
@@ -220,6 +223,8 @@ export class ProjectService {
       postgres,
       mmrStoreType,
       dockerNetwork,
+      ipfsUrl: IPFS_URL,
+      mmrPath,
       ...baseConfig,
       ...advancedConfig,
     };

apps/indexer-coordinator/src/project/types.ts

@@ -21,6 +21,8 @@ export type TemplateType = {
   dbSchema: string;
   postgres: Postgres;
   dockerNetwork: string;
+  ipfsUrl: string;
+  mmrPath: string;
   worker: number;
   batchSize: number;
   timeout: number;

apps/indexer-coordinator/src/utils/project.ts

@@ -4,8 +4,8 @@
 import { IPFSClient } from '@subql/network-clients';
 import yaml from 'js-yaml';
 import { isEqual } from 'lodash';
-
 import { Project, ProjectAdvancedConfig, ProjectBaseConfig } from '../project/project.model';
+import { argv } from '../yargs';
 
 // manifest types
 export type Runner = {
@@ -37,7 +37,7 @@ export type PartialIpfsDeploymentManifest = {
 
 export type ChainType = 'near' | 'flare' | 'cosmos' | 'algorand' | 'substrate' | 'ethereum';
 
-export const IPFS_URL = 'https://authipfs.subquery.network/ipfs/api/v0';
+export const IPFS_URL = argv['ipfs'] ?? 'https://authipfs.subquery.network/ipfs/api/v0';
 const clientSDK = new IPFSClient(IPFS_URL);
 
 export function projectConfigChanged(

apps/indexer-coordinator/src/utils/template.yml

@@ -19,7 +19,7 @@ services:
     command:
       - -f=ipfs://{{deploymentID}}
       - -d={{networkDictionary}}
-      - --ipfs=https://authipfs.subquery.network/ipfs/api/v0
+      - --ipfs={{ipfsUrl}}
       - --network-endpoint={{{networkEndpoint}}}
       - --db-schema={{dbSchema}}
       - --port={{servicePort}}

apps/indexer-coordinator/src/yargs.ts

@@ -82,7 +82,7 @@ function getYargsOption() {
     mmrPath: {
       type: 'string',
       describe: 'The local path to store the mmr data',
-      default: '/home/indexer-service',
+      default: '/home',
       group: Groups.node,
     },
     'docker-network': {