subquery · jiqiang90 · Oct 31, 2023 · Oct 31, 2023 · Oct 31, 2023 · Oct 31, 2023
diff --git a/packages/common-cosmos/CHANGELOG.md b/packages/common-cosmos/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Update `@subql/common` and relevant changes to support endBlock feature (#195)
 
 ## [3.1.1] - 2023-10-26
 ### Changed

diff --git a/packages/common-cosmos/package.json b/packages/common-cosmos/package.json
@@ -23,7 +23,7 @@
     "@protobufs/google": "^0.0.10",
     "@protobufs/ibc": "^0.1.0",
     "@protobufs/tendermint": "^0.0.10",
-    "@subql/common": "^3.2.0",
+    "@subql/common": "^3.3.0",
     "@subql/types-cosmos": "workspace:*",
     "fs-extra": "^11.1.1",
     "js-yaml": "^4.1.0",

diff --git a/packages/common-cosmos/src/project/models.ts b/packages/common-cosmos/src/project/models.ts
@@ -1,7 +1,7 @@
 // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {ProcessorImpl} from '@subql/common';
+import {BaseDataSource, ProcessorImpl} from '@subql/common';
 import {FileReference, Processor} from '@subql/types-core';
 import {
   CosmosEventFilter,
@@ -22,7 +22,7 @@
  CosmosMessageHandler,
  CustomModule,
  CosmosTxFilter,
  SubqlCosmosProcessorOptions,
 } from '@subql/types-cosmos';
 import {plainToClass, Transform, Type} from 'class-transformer';
 import {
@@ -140,7 +140,7 @@
        case CosmosHandlerKind.Block:
          return plainToClass(BlockHandler, handler);
        default:
          throw new Error(`handler ${(handler as any).kind} not supported`);
      }
    });
  })
@@ -167,15 +167,14 @@
 }
 
 export class CosmosRuntimeDataSourceBase<M extends CosmosMapping<CosmosRuntimeHandler>>
+  extends BaseDataSource
   implements CosmosRuntimeDatasource<M>
 {
   @IsEnum(CosmosDatasourceKind, {groups: [CosmosDatasourceKind.Runtime]})
   kind: CosmosDatasourceKind.Runtime;
   @Type(() => RuntimeMapping)
   @ValidateNested()
   mapping: M;
-  @IsInt()
-  startBlock: number;
   @IsOptional()
   @Validate(FileReferenceImp)
   assets?: Map<string, FileReference>;
@@ -199,19 +198,18 @@
 }
 
 export class CosmosCustomDataSourceBase<
-  K extends string,
-  M extends CosmosMapping = CosmosMapping<CosmosCustomHandler>,
-  O = any
-> implements CosmosCustomDatasource<K, M, O>
+    K extends string,
+    M extends CosmosMapping = CosmosMapping<CosmosCustomHandler>,
+    O = any
+  >
+  extends BaseDataSource
+  implements CosmosCustomDatasource<K, M, O>
 {
   @IsString()
   kind: K;
   @Type(() => CustomMapping)
   @ValidateNested()
   mapping: M;
-  @IsOptional()
-  @IsInt()
-  startBlock?: number;
   @Type(() => CosmosFileReferenceImpl)
   @ValidateNested({each: true})
   assets: Map<string, CustomDataSourceAsset>;

diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Update `@subql/node-core` with fixes and support for endBlock feature (#195)
 ### Fixed
 - Missing dependencies for testing command (#194)
 - Events getting processed multiple times (#194)

diff --git a/packages/node/package.json b/packages/node/package.json
@@ -28,9 +28,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^3.2.0",
+    "@subql/common": "^3.3.0",
     "@subql/common-cosmos": "workspace:*",
-    "@subql/node-core": "^6.1.1",
+    "@subql/node-core": "^6.2.0",
     "@subql/types-cosmos": "workspace:*",
     "cosmjs-types": "^0.7.0",
     "cron-converter": "^1.0.2",

diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts
@@ -6,30 +6,18 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   NodeConfig,
-  Worker,
   SmartBatchService,
   StoreService,
   PoiService,
   PoiSyncService,
   StoreCacheService,
   IProjectService,
-  IDynamicDsService,
-  HostStore,
-  HostDynamicDS,
   WorkerBlockDispatcher,
-  IUnfinalizedBlocksService,
-  HostConnectionPoolState,
-  connectionPoolStateHostFunctions,
   ConnectionPoolStateManager,
-  HostUnfinalizedBlocks,
-  baseWorkerFunctions,
-  storeHostFunctions,
-  cacheHostFunctions,
-  dynamicDsHostFunctions,
   IProjectUpgradeService,
   InMemoryCacheService,
+  createIndexerWorker,
 } from '@subql/node-core';
-import { Cache, Store } from '@subql/types-core';
 import {
   CosmosProjectDs,
   SubqueryProject,
@@ -38,48 +26,12 @@ import { CosmosClientConnection } from '../cosmosClient.connection';
 import { DynamicDsService } from '../dynamic-ds.service';
 import { BlockContent } from '../types';
 import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service';
-import { IIndexerWorker, IInitIndexerWorker } from '../worker/worker';
+import { IIndexerWorker } from '../worker/worker';
 
 type IndexerWorker = IIndexerWorker & {
   terminate: () => Promise<number>;
 };
 
-async function createIndexerWorker(
-  store: Store,
-  cache: Cache,
-  dynamicDsService: IDynamicDsService<CosmosProjectDs>,
-  unfinalizedBlocksService: IUnfinalizedBlocksService<BlockContent>,
-  connectionPoolState: ConnectionPoolStateManager<CosmosClientConnection>,
-  root: string,
-  startHeight: number,
-): Promise<IndexerWorker> {
-  const indexerWorker = Worker.create<
-    IInitIndexerWorker,
-    HostDynamicDS<CosmosProjectDs> &
-      HostStore &
-      HostUnfinalizedBlocks &
-      HostConnectionPoolState<CosmosClientConnection>
-  >(
-    path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),
-    [...baseWorkerFunctions, 'initWorker'],
-    {
-      ...cacheHostFunctions(cache),
-      ...storeHostFunctions(store),
-      ...dynamicDsHostFunctions(dynamicDsService),
-      unfinalizedBlocksProcess:
-        unfinalizedBlocksService.processUnfinalizedBlockHeader.bind(
-          unfinalizedBlocksService,
-        ),
-      ...connectionPoolStateHostFunctions(connectionPoolState),
-    },
-    root,
-  );
-
-  await indexerWorker.initWorker(startHeight);
-
-  return indexerWorker;
-}
-
 @Injectable()
 export class WorkerBlockDispatcherService
   extends WorkerBlockDispatcher<CosmosProjectDs, IndexerWorker>
@@ -115,7 +67,14 @@ export class WorkerBlockDispatcherService
       project,
       dynamicDsService,
       () =>
-        createIndexerWorker(
+        createIndexerWorker<
+          IIndexerWorker,
+          CosmosClientConnection,
+          BlockContent,
+          CosmosProjectDs
+        >(
+          path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),
+          [],
           storeService.getStore(),
           cacheService.getCache(),
           dynamicDsService,

diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts
@@ -9,6 +9,7 @@ import {
   ConnectionPoolStateManager,
   WorkerConnectionPoolStateManager,
   InMemoryCacheService,
+  WorkerInMemoryCacheService,
 } from '@subql/node-core';
 import { SubqueryProject } from '../../configure/SubqueryProject';
 import { ApiService } from '../api.service';
@@ -19,7 +20,6 @@ import { IndexerManager } from '../indexer.manager';
 import { ProjectService } from '../project.service';
 import { SandboxService } from '../sandbox.service';
 import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service';
-import { WorkerInMemoryCacheService } from './worker.inMemoryCache.service';
 import { WorkerService } from './worker.service';
 import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.service';
 

diff --git a/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts b/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts
diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts
@@ -55,10 +55,6 @@ async function initWorker(startHeight: number): Promise<void> {
 
 export type IIndexerWorker = IBaseIndexerWorker;
 
-export type IInitIndexerWorker = IIndexerWorker & {
-  initWorker: typeof initWorker;
-};
-
 (global as any).host = createWorkerHost([], {
   initWorker,
 });
diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts
@@ -11,10 +11,10 @@ import {
   InMemoryCacheService,
   PoiService,
   PoiSyncService,
+  StoreCacheService,
   StoreService,
   TestRunner,
 } from '@subql/node-core';
-import { StoreCacheService } from '@subql/node-core/dist';
 import { ConfigureModule } from '../configure/configure.module';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { ApiService } from '../indexer/api.service';

diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts
@@ -82,7 +82,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
     },
     debug: {
       demandOption: false,
-      describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary"`,
+      describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary". To disable specific scopes you can prefix them with '-'. e.g. "*,-SQL"`,
       type: 'string',
     },
     'dictionary-resolver': {

diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Import `@subql/types-core` global into global so its no longer needed to update tsconfig in projects (#195)
 ### Added
 - JSDoc to field explaining definition (#194)
 

diff --git a/packages/types/package.json b/packages/types/package.json
@@ -19,7 +19,7 @@
     "@cosmjs/cosmwasm-stargate": "^0.30.1",
     "@cosmjs/proto-signing": "^0.30.1",
     "@cosmjs/stargate": "^0.30.1",
-    "@subql/types-core": "^0.2.0"
+    "@subql/types-core": "^0.3.0"
   },
   "stableVersion": "3.1.0"
 }
diff --git a/packages/types/src/global.ts b/packages/types/src/global.ts
@@ -3,6 +3,7 @@
 
 import type {CosmWasmClient} from '@cosmjs/cosmwasm-stargate';
 import {CosmWasmSafeClient} from './interfaces';
+import '@subql/types-core/dist/global';
 
 declare global {
   const apiUnsafe: CosmWasmClient; //requires --unsafe flag to be defined