subquery · jiqiang90 · Oct 12, 2023 · Oct 11, 2023 · Oct 12, 2023
diff --git a/packages/common-cosmos/CHANGELOG.md b/packages/common-cosmos/CHANGELOG.md
@@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+### Changed
+- Version bump with `@subql/common` 3.1.2
+
 ## [3.0.1] - 2023-10-05
 ### Update
 - Bump `subql/types-cosmos`

diff --git a/packages/common-cosmos/package.json b/packages/common-cosmos/package.json
@@ -22,7 +22,7 @@
     "@protobufs/google": "^0.0.10",
     "@protobufs/ibc": "^0.1.0",
     "@protobufs/tendermint": "^0.0.10",
-    "@subql/common": "^3.1.0",
+    "@subql/common": "^3.1.2",
     "@subql/types-cosmos": "workspace:*",
     "fs-extra": "^11.1.1",
     "js-yaml": "^4.1.0",

diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md
@@ -5,6 +5,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+### Changed
+- debug has changed from a boolean to a string to allow scoping debug log level (#2077)
+
+### Fixed
+- Sync with node-core.
+  - Fixed Poi migration performance issue.
+  - Fixed AutoQueue timeout issue.
+  - Fixed Poi sync could block DB IO and drop connection issue.
+
+
 ## [3.0.1] - 2023-10-05
 ### Update
 - Bump `subql/types-cosmos`

diff --git a/packages/node/package.json b/packages/node/package.json
@@ -27,9 +27,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^3.1.0",
+    "@subql/common": "^3.1.2",
     "@subql/common-cosmos": "workspace:*",
-    "@subql/node-core": "^5.0.2",
+    "@subql/node-core": "^6.0.1",
     "@subql/types-cosmos": "workspace:*",
     "cosmjs-types": "^0.7.0",
     "cron-converter": "^1.0.2",

diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts
@@ -10,6 +10,7 @@ import {
   StoreService,
   IProjectService,
   PoiService,
+  PoiSyncService,
   BlockDispatcher,
   ProcessBlockResponse,
   IProjectUpgradeService,
@@ -43,6 +44,7 @@ export class BlockDispatcherService
     storeService: StoreService,
     storeCacheService: StoreCacheService,
     poiService: PoiService,
+    poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
     dynamicDsService: DynamicDsService,
   ) {
@@ -55,6 +57,7 @@ export class BlockDispatcherService
       storeService,
       storeCacheService,
       poiService,
+      poiSyncService,
       project,
       dynamicDsService,
       async (blockNums: number[]): Promise<BlockContent[]> => {

diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts
@@ -10,6 +10,7 @@ import {
   SmartBatchService,
   StoreService,
   PoiService,
+  PoiSyncService,
   StoreCacheService,
   IProjectService,
   IDynamicDsService,
@@ -90,6 +91,7 @@ export class WorkerBlockDispatcherService
     storeService: StoreService,
     storeCacheService: StoreCacheService,
     poiService: PoiService,
+    poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
     dynamicDsService: DynamicDsService,
     unfinalizedBlocksSevice: UnfinalizedBlocksService,
@@ -104,6 +106,7 @@ export class WorkerBlockDispatcherService
       storeService,
       storeCacheService,
       poiService,
+      poiSyncService,
       project,
       dynamicDsService,
       () =>

diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts
@@ -8,6 +8,7 @@ import {
   IndexingBenchmarkService,
   StoreService,
   PoiService,
+  PoiSyncService,
   NodeConfig,
   ConnectionPoolService,
   SmartBatchService,
@@ -58,6 +59,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         storeService: StoreService,
         storeCacheService: StoreCacheService,
         poiService: PoiService,
+        poiSyncService: PoiSyncService,
         project: SubqueryProject,
         dynamicDsService: DynamicDsService,
         unfinalizedBlocks: UnfinalizedBlocksService,
@@ -73,6 +75,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               storeService,
               storeCacheService,
               poiService,
+              poiSyncService,
               project,
               dynamicDsService,
               unfinalizedBlocks,
@@ -89,6 +92,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               storeService,
               storeCacheService,
               poiService,
+              poiSyncService,
               project,
               dynamicDsService,
             ),
@@ -103,6 +107,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         StoreService,
         StoreCacheService,
         PoiService,
+        PoiSyncService,
         'ISubqueryProject',
         DynamicDsService,
         UnfinalizedBlocksService,
@@ -118,6 +123,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
     DsProcessorService,
     DynamicDsService,
     PoiService,
+    PoiSyncService,
     {
       useClass: ProjectService,
       provide: 'IProjectService',

diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts
@@ -9,6 +9,7 @@ import {
   NodeConfig,
   StoreService,
   PoiService,
+  PoiSyncService,
   BaseProjectService,
   IProjectUpgradeService,
 } from '@subql/node-core';
@@ -33,6 +34,8 @@ export class ProjectService extends BaseProjectService<
     dsProcessorService: DsProcessorService,
     apiService: ApiService,
     @Inject(isMainThread ? PoiService : 'Null') poiService: PoiService,
+    @Inject(isMainThread ? PoiSyncService : 'Null')
+    poiSyncService: PoiSyncService,
     @Inject(isMainThread ? Sequelize : 'Null') sequelize: Sequelize,
     @Inject('ISubqueryProject') project: SubqueryProject,
     @Inject('IProjectUpgradeService')
@@ -47,6 +50,7 @@ export class ProjectService extends BaseProjectService<
       dsProcessorService,
       apiService,
       poiService,
+      poiSyncService,
       sequelize,
       project,
       projectUpgradeService,

diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts
@@ -34,7 +34,7 @@ const logger = getLogger(`worker #${threadId}`);
 async function initWorker(startHeight: number): Promise<void> {
   try {
     const app = await NestFactory.create(WorkerModule, {
-      logger: new NestLogger(argv.debug), // TIP: If the worker is crashing comment out this line for better logging
+      logger: new NestLogger(!!argv.debug), // TIP: If the worker is crashing comment out this line for better logging
     });
 
     await app.init();

diff --git a/packages/node/src/init.ts b/packages/node/src/init.ts
@@ -20,7 +20,6 @@ notifyUpdates(pjson, logger);
 
 export async function bootstrap(): Promise<void> {
   logger.info(`Current ${pjson.name} version is ${pjson.version}`);
-  const debug = argv.debug;
 
   const validate = (x: any) => {
     const p = parseInt(x);
@@ -45,7 +44,7 @@ export async function bootstrap(): Promise<void> {
 
   try {
     const app = await NestFactory.create(AppModule, {
-      logger: new NestLogger(debug),
+      logger: new NestLogger(!!argv.debug),
     });
     await app.init();
 

diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts
@@ -13,7 +13,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
     builder: {},
     handler: (argv) => {
       initLogger(
-        argv.debug as boolean,
+        argv.debug as string,
         argv.outputFmt as 'json' | 'colored',
         argv.logLevel as string | undefined,
       );
@@ -30,7 +30,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
     builder: {},
     handler: (argv) => {
       initLogger(
-        argv.debug as boolean,
+        argv.debug as string,
         argv.outputFmt as 'json' | 'colored',
         argv.logLevel as string | undefined,
       );
@@ -82,10 +82,8 @@ export const yargsOptions = yargs(hideBin(process.argv))
     },
     debug: {
       demandOption: false,
-      describe:
-        'Show debug information to console output. will forcefully set log level to debug',
-      type: 'boolean',
-      default: false,
+      describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary"`,
+      type: 'string',
     },
     'dictionary-resolver': {
       demandOption: false,

diff --git a/packages/types/package.json b/packages/types/package.json
@@ -18,6 +18,6 @@
     "@cosmjs/cosmwasm-stargate": "^0.30.1",
     "@cosmjs/proto-signing": "^0.30.1",
     "@cosmjs/stargate": "^0.30.1",
-    "@subql/types-core": "^0.1.0"
+    "@subql/types-core": "^0.1.1"
   }
 }
diff --git a/yarn.lock b/yarn.lock
@@ -4096,7 +4096,7 @@ __metadata:
     "@protobufs/google": ^0.0.10
     "@protobufs/ibc": ^0.1.0
     "@protobufs/tendermint": ^0.0.10
-    "@subql/common": ^3.1.0
+    "@subql/common": ^3.1.2
     "@subql/types-cosmos": "workspace:*"
     "@types/bn.js": 4.11.6
     "@types/ejs": ^3.1.2
@@ -4118,11 +4118,11 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@subql/common@npm:3.1.0, @subql/common@npm:^3.1.0":
-  version: 3.1.0
-  resolution: "@subql/common@npm:3.1.0"
+"@subql/common@npm:3.1.2, @subql/common@npm:^3.1.2":
+  version: 3.1.2
+  resolution: "@subql/common@npm:3.1.2"
   dependencies:
-    "@subql/types-core": 0.1.0
+    "@subql/types-core": 0.1.1
     axios: ^0.27.2
     class-transformer: ^0.5.1
     class-validator: ^0.14.0
@@ -4132,23 +4132,23 @@ __metadata:
     reflect-metadata: ^0.1.13
     semver: ^7.5.2
     update-notifier: 5.1.0
-  checksum: 43c7207acbc943cf1196c6ea0ecc6187b5a0a08a1f358861da5c2fa2db1b1a94ad6232850ef85e0c1a49274fd96d5c39204d860aa18d0e655fcad0b402df0220
+  checksum: e979fac54089276de67a3a498bdf53396ac3726542d9f38350bced06f45921ab937a885a9c6cf183f3f5e7c82877c8ce9e27a5a5bf1f6fae6113baf68b4b3e55
   languageName: node
   linkType: hard
 
-"@subql/node-core@npm:^5.0.2":
-  version: 5.0.2
-  resolution: "@subql/node-core@npm:5.0.2"
+"@subql/node-core@npm:^6.0.1":
+  version: 6.0.1
+  resolution: "@subql/node-core@npm:6.0.1"
   dependencies:
     "@apollo/client": ^3.7.16
     "@nestjs/common": ^9.4.0
     "@nestjs/event-emitter": ^2.0.0
     "@nestjs/schedule": ^3.0.1
     "@subql/apollo-links": ^1.0.2
-    "@subql/common": 3.1.0
+    "@subql/common": 3.1.2
     "@subql/testing": 2.0.3-0
-    "@subql/types": 3.1.0
-    "@subql/utils": 2.4.4-1
+    "@subql/types": 3.1.3-0
+    "@subql/utils": 2.4.4
     "@subql/x-sequelize": 6.32.0-0.0.2
     "@willsoto/nestjs-prometheus": ^5.4.0
     async-lock: ^1.4.0
@@ -4163,7 +4163,7 @@ __metadata:
     tar: ^6.1.11
     vm2: ^3.9.19
     yargs: ^16.2.0
-  checksum: e746752c239b4cdd9426ce664cee4330ca38e8bf8882646637404cecdb5f3e1ec846255ab5b3e19127de0c7c0d27ae22983400699b71b2a25bac473b1cb057a0
+  checksum: 8c118db94ce2756643092ff71b15eb30ec46256cd6bbbd2beefc8f2cae53d7800f9f4d97d2a045d68a7621040c6399da9902717ed25dddd5fea3334d2f2cdbd0
   languageName: node
   linkType: hard
 
@@ -4182,9 +4182,9 @@ __metadata:
     "@nestjs/schedule": ^3.0.1
     "@nestjs/schematics": ^9.2.0
     "@nestjs/testing": ^9.4.0
-    "@subql/common": ^3.1.0
+    "@subql/common": ^3.1.2
     "@subql/common-cosmos": "workspace:*"
-    "@subql/node-core": ^5.0.2
+    "@subql/node-core": ^6.0.1
     "@subql/types-cosmos": "workspace:*"
     "@types/express": ^4.17.13
     "@types/jest": ^27.4.0
@@ -4221,12 +4221,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@subql/types-core@npm:0.1.0":
-  version: 0.1.0
-  resolution: "@subql/types-core@npm:0.1.0"
+"@subql/types-core@npm:0.1.1, @subql/types-core@npm:^0.1.1":
+  version: 0.1.1
+  resolution: "@subql/types-core@npm:0.1.1"
   dependencies:
     package-json-type: ^1.0.3
-  checksum: fd86c2cf5148a4ad3af5c3a4fa7f0180f7a4d90fd3f6d96bc0e48eab6d73767220c859c751b21bbe08565d31eac5e8d0cee0148dee7ee7e0e273ff46182d3131
+  checksum: 6a88547e5091795d2f9f24b5373ce531066bc04602b18f05cad77c5d1953523be479801447cd85e4f03468997e988b8c853fdb409631f8a396525cc723978afa
   languageName: node
   linkType: hard
 
@@ -4239,23 +4239,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@subql/types-core@npm:^0.1.0":
-  version: 0.1.1
-  resolution: "@subql/types-core@npm:0.1.1"
-  dependencies:
-    package-json-type: ^1.0.3
-  checksum: 6a88547e5091795d2f9f24b5373ce531066bc04602b18f05cad77c5d1953523be479801447cd85e4f03468997e988b8c853fdb409631f8a396525cc723978afa
-  languageName: node
-  linkType: hard
-
 "@subql/types-cosmos@workspace:*, @subql/types-cosmos@workspace:packages/types":
   version: 0.0.0-use.local
   resolution: "@subql/types-cosmos@workspace:packages/types"
   dependencies:
     "@cosmjs/cosmwasm-stargate": ^0.30.1
     "@cosmjs/proto-signing": ^0.30.1
     "@cosmjs/stargate": ^0.30.1
-    "@subql/types-core": ^0.1.0
+    "@subql/types-core": ^0.1.1
   languageName: unknown
   linkType: soft
 
@@ -4268,9 +4259,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@subql/utils@npm:2.4.4-1":
-  version: 2.4.4-1
-  resolution: "@subql/utils@npm:2.4.4-1"
+"@subql/utils@npm:2.4.4":
+  version: 2.4.4
+  resolution: "@subql/utils@npm:2.4.4"
   dependencies:
     "@polkadot/util": ^12.2.1
     "@polkadot/util-crypto": ^12.2.1
@@ -4287,7 +4278,7 @@ __metadata:
     rotating-file-stream: ^3.0.2
     semver: ^7.5.2
     tar: ^6.1.11
-  checksum: f80fac522d9739a493b8afe8f69029996b1e597241a60c8e2641a6192da1a03300c76738f9bcdaedc973f208ed60c6e9608483e60dd8711b0d0843d42b647b42
+  checksum: 1b0a4a4435bddc2bb058283a4417611be5b8c0248a99dc9f5ea32a1255b60beb64217e49e6bf1c99fef14625e9fa561bdf5a481e3208229093cdca8dfd59a71f
   languageName: node
   linkType: hard