From c1cbeab15a8d5032efe1c20c4b0f2b2a1f75a4e7 Mon Sep 17 00:00:00 2001 From: stwiname Date: Tue, 31 Oct 2023 05:05:20 +0000 Subject: [PATCH 1/3] Update @subql deps --- packages/common-cosmos/package.json | 2 +- packages/node/package.json | 4 +-- packages/types/package.json | 2 +- yarn.lock | 48 ++++++++++++++--------------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/packages/common-cosmos/package.json b/packages/common-cosmos/package.json index 906ccee9f..c2039d3fe 100644 --- a/packages/common-cosmos/package.json +++ b/packages/common-cosmos/package.json @@ -23,7 +23,7 @@ "@protobufs/google": "^0.0.10", "@protobufs/ibc": "^0.1.0", "@protobufs/tendermint": "^0.0.10", - "@subql/common": "^3.2.0", + "@subql/common": "^3.3.0", "@subql/types-cosmos": "workspace:*", "fs-extra": "^11.1.1", "js-yaml": "^4.1.0", diff --git a/packages/node/package.json b/packages/node/package.json index 1d4d6b403..7aa013903 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -28,9 +28,9 @@ "@nestjs/event-emitter": "^2.0.0", "@nestjs/platform-express": "^9.4.0", "@nestjs/schedule": "^3.0.1", - "@subql/common": "^3.2.0", + "@subql/common": "^3.3.0", "@subql/common-cosmos": "workspace:*", - "@subql/node-core": "^6.1.1", + "@subql/node-core": "^6.2.0", "@subql/types-cosmos": "workspace:*", "cosmjs-types": "^0.7.0", "cron-converter": "^1.0.2", diff --git a/packages/types/package.json b/packages/types/package.json index fbb29ca91..80eb283b5 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -19,7 +19,7 @@ "@cosmjs/cosmwasm-stargate": "^0.30.1", "@cosmjs/proto-signing": "^0.30.1", "@cosmjs/stargate": "^0.30.1", - "@subql/types-core": "^0.2.0" + "@subql/types-core": "^0.3.0" }, "stableVersion": "3.1.0" } diff --git a/yarn.lock b/yarn.lock index c341feda5..d146e7af0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4251,7 +4251,7 @@ __metadata: "@protobufs/google": ^0.0.10 "@protobufs/ibc": ^0.1.0 "@protobufs/tendermint": ^0.0.10 - "@subql/common": ^3.2.0 + "@subql/common": ^3.3.0 "@subql/types-cosmos": "workspace:*" "@types/bn.js": 4.11.6 "@types/ejs": ^3.1.2 @@ -4273,11 +4273,11 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:3.2.0, @subql/common@npm:^3.2.0": - version: 3.2.0 - resolution: "@subql/common@npm:3.2.0" +"@subql/common@npm:3.3.0, @subql/common@npm:^3.3.0": + version: 3.3.0 + resolution: "@subql/common@npm:3.3.0" dependencies: - "@subql/types-core": 0.2.0 + "@subql/types-core": 0.3.0 axios: ^0.27.2 class-transformer: ^0.5.1 class-validator: ^0.14.0 @@ -4287,23 +4287,23 @@ __metadata: reflect-metadata: ^0.1.13 semver: ^7.5.2 update-notifier: 5.1.0 - checksum: bddf220f66e008a6b1ea40081143cc7ad71e144597a9a44cb59acb865923746006d13cfa376b49794fa81ca14200e905f5bea7292f6d70416353fdbe9f009dc1 + checksum: 50d904c0c7e0b8caeb1f53a968c9c5aa223fa8bc822ab164edbe7043a922e5c200d1ac67996358d9152d443e9288c3e5164c906181cc305a0471ffa6c8e2fdea languageName: node linkType: hard -"@subql/node-core@npm:^6.1.1": - version: 6.1.1 - resolution: "@subql/node-core@npm:6.1.1" +"@subql/node-core@npm:^6.2.0": + version: 6.2.0 + resolution: "@subql/node-core@npm:6.2.0" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 "@subql/apollo-links": ^1.0.2 - "@subql/common": 3.2.0 + "@subql/common": 3.3.0 "@subql/testing": 2.1.0 - "@subql/types": 3.2.0 - "@subql/utils": 2.4.5-0 + "@subql/types": 3.3.0 + "@subql/utils": 2.5.0 "@subql/x-sequelize": 6.32.0-0.0.2 "@willsoto/nestjs-prometheus": ^5.4.0 async-lock: ^1.4.0 @@ -4318,7 +4318,7 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: 7709de62a75d2461c185d88600be657c43d9a8b0ab90355d5cece694b0077c57a329f4c5963f29213e0b0da117141c09c965561219cdc26614015441b99c9158 + checksum: ec4deffd88a94067a634349b8b809415a3d0dbc28a3c187df8482409fedef85ce343f497ff68e0c0b86ea0520f4864aee306986a62be0e4e20df14920ec66b40 languageName: node linkType: hard @@ -4337,9 +4337,9 @@ __metadata: "@nestjs/schedule": ^3.0.1 "@nestjs/schematics": ^9.2.0 "@nestjs/testing": ^9.4.0 - "@subql/common": ^3.2.0 + "@subql/common": ^3.3.0 "@subql/common-cosmos": "workspace:*" - "@subql/node-core": ^6.1.1 + "@subql/node-core": ^6.2.0 "@subql/types-cosmos": "workspace:*" "@types/express": ^4.17.13 "@types/jest": ^27.4.0 @@ -4376,12 +4376,12 @@ __metadata: languageName: node linkType: hard -"@subql/types-core@npm:0.2.0, @subql/types-core@npm:^0.2.0": - version: 0.2.0 - resolution: "@subql/types-core@npm:0.2.0" +"@subql/types-core@npm:0.3.0, @subql/types-core@npm:^0.3.0": + version: 0.3.0 + resolution: "@subql/types-core@npm:0.3.0" dependencies: package-json-type: ^1.0.3 - checksum: ef538b84cb0c32f41269fdb6708e9e0657da9e7f0469ef24882e0b392a797eaf35364a7ef9368c0c89b799d6522a66cbf17781deae1edfdc8a17f9cd43aed95f + checksum: fa9ec3bb27d33b57a18425553f6f9239dca439f37f296ae49dd1da958a3d6f14d73aa0831c057828618798926049c0308f3cba68276c80bfbff2fbfe265d1bf3 languageName: node linkType: hard @@ -4401,7 +4401,7 @@ __metadata: "@cosmjs/cosmwasm-stargate": ^0.30.1 "@cosmjs/proto-signing": ^0.30.1 "@cosmjs/stargate": ^0.30.1 - "@subql/types-core": ^0.2.0 + "@subql/types-core": ^0.3.0 languageName: unknown linkType: soft @@ -4414,9 +4414,9 @@ __metadata: languageName: node linkType: hard -"@subql/utils@npm:2.4.5-0": - version: 2.4.5-0 - resolution: "@subql/utils@npm:2.4.5-0" +"@subql/utils@npm:2.5.0": + version: 2.5.0 + resolution: "@subql/utils@npm:2.5.0" dependencies: "@polkadot/util": ^12.2.1 "@polkadot/util-crypto": ^12.2.1 @@ -4433,7 +4433,7 @@ __metadata: rotating-file-stream: ^3.0.2 semver: ^7.5.2 tar: ^6.1.11 - checksum: 7d76203fcb25251225ce7a2a56bf3de8e2a98caed4f1598ca50dea4bf96511bec71b7d4bdbec8648a7b76b654259a94dbdf2c66105961e8aebe780c318ad2ae0 + checksum: 27d958f916f519aba9257d5bb4d310f0bb08d628098643c417342fbf864e6138de28d7fd0c8a3b3177f93668544b0655a6b483c9b1fe5fabbd8b08babc2ea36a languageName: node linkType: hard From 488e3f64b06880b802df909aca781f497f4b85f1 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Tue, 31 Oct 2023 17:27:35 +1300 Subject: [PATCH 2/3] Apply changes from main sdk --- packages/common-cosmos/src/project/models.ts | 18 +++--- .../worker-block-dispatcher.service.ts | 61 +++---------------- .../src/indexer/worker/worker-fetch.module.ts | 2 +- .../worker/worker.inMemoryCache.service.ts | 20 ------ packages/node/src/indexer/worker/worker.ts | 4 -- .../node/src/subcommands/testing.module.ts | 2 +- packages/node/src/yargs.ts | 2 +- packages/types/src/global.ts | 1 + 8 files changed, 22 insertions(+), 88 deletions(-) delete mode 100644 packages/node/src/indexer/worker/worker.inMemoryCache.service.ts diff --git a/packages/common-cosmos/src/project/models.ts b/packages/common-cosmos/src/project/models.ts index bb5d01ddf..6e54c2f37 100644 --- a/packages/common-cosmos/src/project/models.ts +++ b/packages/common-cosmos/src/project/models.ts @@ -1,7 +1,7 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {ProcessorImpl} from '@subql/common'; +import {BaseDataSource, ProcessorImpl} from '@subql/common'; import {FileReference, Processor} from '@subql/types-core'; import { CosmosEventFilter, @@ -167,6 +167,7 @@ export class CosmosProcessorOptions implements CosmosProcessorOptions { } export class CosmosRuntimeDataSourceBase> + extends BaseDataSource implements CosmosRuntimeDatasource { @IsEnum(CosmosDatasourceKind, {groups: [CosmosDatasourceKind.Runtime]}) @@ -174,8 +175,6 @@ export class CosmosRuntimeDataSourceBase RuntimeMapping) @ValidateNested() mapping: M; - @IsInt() - startBlock: number; @IsOptional() @Validate(FileReferenceImp) assets?: Map; @@ -199,19 +198,18 @@ export class CosmosCustomModuleImpl implements CustomModule { } export class CosmosCustomDataSourceBase< - K extends string, - M extends CosmosMapping = CosmosMapping, - O = any -> implements CosmosCustomDatasource + K extends string, + M extends CosmosMapping = CosmosMapping, + O = any + > + extends BaseDataSource + implements CosmosCustomDatasource { @IsString() kind: K; @Type(() => CustomMapping) @ValidateNested() mapping: M; - @IsOptional() - @IsInt() - startBlock?: number; @Type(() => CosmosFileReferenceImpl) @ValidateNested({each: true}) assets: Map; diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index 646caac38..19137d564 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -6,30 +6,18 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { NodeConfig, - Worker, SmartBatchService, StoreService, PoiService, PoiSyncService, StoreCacheService, IProjectService, - IDynamicDsService, - HostStore, - HostDynamicDS, WorkerBlockDispatcher, - IUnfinalizedBlocksService, - HostConnectionPoolState, - connectionPoolStateHostFunctions, ConnectionPoolStateManager, - HostUnfinalizedBlocks, - baseWorkerFunctions, - storeHostFunctions, - cacheHostFunctions, - dynamicDsHostFunctions, IProjectUpgradeService, InMemoryCacheService, + createIndexerWorker, } from '@subql/node-core'; -import { Cache, Store } from '@subql/types-core'; import { CosmosProjectDs, SubqueryProject, @@ -38,48 +26,12 @@ import { CosmosClientConnection } from '../cosmosClient.connection'; import { DynamicDsService } from '../dynamic-ds.service'; import { BlockContent } from '../types'; import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; -import { IIndexerWorker, IInitIndexerWorker } from '../worker/worker'; +import { IIndexerWorker } from '../worker/worker'; type IndexerWorker = IIndexerWorker & { terminate: () => Promise; }; -async function createIndexerWorker( - store: Store, - cache: Cache, - dynamicDsService: IDynamicDsService, - unfinalizedBlocksService: IUnfinalizedBlocksService, - connectionPoolState: ConnectionPoolStateManager, - root: string, - startHeight: number, -): Promise { - const indexerWorker = Worker.create< - IInitIndexerWorker, - HostDynamicDS & - HostStore & - HostUnfinalizedBlocks & - HostConnectionPoolState - >( - path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'), - [...baseWorkerFunctions, 'initWorker'], - { - ...cacheHostFunctions(cache), - ...storeHostFunctions(store), - ...dynamicDsHostFunctions(dynamicDsService), - unfinalizedBlocksProcess: - unfinalizedBlocksService.processUnfinalizedBlockHeader.bind( - unfinalizedBlocksService, - ), - ...connectionPoolStateHostFunctions(connectionPoolState), - }, - root, - ); - - await indexerWorker.initWorker(startHeight); - - return indexerWorker; -} - @Injectable() export class WorkerBlockDispatcherService extends WorkerBlockDispatcher @@ -115,7 +67,14 @@ export class WorkerBlockDispatcherService project, dynamicDsService, () => - createIndexerWorker( + createIndexerWorker< + IIndexerWorker, + CosmosClientConnection, + BlockContent, + CosmosProjectDs + >( + path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'), + [], storeService.getStore(), cacheService.getCache(), dynamicDsService, diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts index 4d394da70..ab2ba0d5f 100644 --- a/packages/node/src/indexer/worker/worker-fetch.module.ts +++ b/packages/node/src/indexer/worker/worker-fetch.module.ts @@ -9,6 +9,7 @@ import { ConnectionPoolStateManager, WorkerConnectionPoolStateManager, InMemoryCacheService, + WorkerInMemoryCacheService, } from '@subql/node-core'; import { SubqueryProject } from '../../configure/SubqueryProject'; import { ApiService } from '../api.service'; @@ -19,7 +20,6 @@ import { IndexerManager } from '../indexer.manager'; import { ProjectService } from '../project.service'; import { SandboxService } from '../sandbox.service'; import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; -import { WorkerInMemoryCacheService } from './worker.inMemoryCache.service'; import { WorkerService } from './worker.service'; import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.service'; diff --git a/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts b/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts deleted file mode 100644 index 1011f9545..000000000 --- a/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { isMainThread } from 'worker_threads'; -import { Injectable } from '@nestjs/common'; -import { HostCache, hostCacheToCache } from '@subql/node-core'; -import { Cache } from '@subql/types-core'; - -@Injectable() -export class WorkerInMemoryCacheService { - constructor(private host: HostCache) { - if (isMainThread) { - throw new Error('Expected to be worker thread'); - } - } - - getCache(): Cache { - return hostCacheToCache(this.host); - } -} diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts index b01e52a00..b72da24b4 100644 --- a/packages/node/src/indexer/worker/worker.ts +++ b/packages/node/src/indexer/worker/worker.ts @@ -55,10 +55,6 @@ async function initWorker(startHeight: number): Promise { export type IIndexerWorker = IBaseIndexerWorker; -export type IInitIndexerWorker = IIndexerWorker & { - initWorker: typeof initWorker; -}; - (global as any).host = createWorkerHost([], { initWorker, }); diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts index 336c8e1cc..e16f05753 100644 --- a/packages/node/src/subcommands/testing.module.ts +++ b/packages/node/src/subcommands/testing.module.ts @@ -11,10 +11,10 @@ import { InMemoryCacheService, PoiService, PoiSyncService, + StoreCacheService, StoreService, TestRunner, } from '@subql/node-core'; -import { StoreCacheService } from '@subql/node-core/dist'; import { ConfigureModule } from '../configure/configure.module'; import { SubqueryProject } from '../configure/SubqueryProject'; import { ApiService } from '../indexer/api.service'; diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts index c6ea8a40f..82421565d 100644 --- a/packages/node/src/yargs.ts +++ b/packages/node/src/yargs.ts @@ -82,7 +82,7 @@ export const yargsOptions = yargs(hideBin(process.argv)) }, debug: { demandOption: false, - describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary"`, + describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary". To disable specific scopes you can prefix them with '-'. e.g. "*,-SQL"`, type: 'string', }, 'dictionary-resolver': { diff --git a/packages/types/src/global.ts b/packages/types/src/global.ts index df9e4ea36..84b2c5e72 100644 --- a/packages/types/src/global.ts +++ b/packages/types/src/global.ts @@ -3,6 +3,7 @@ import type {CosmWasmClient} from '@cosmjs/cosmwasm-stargate'; import {CosmWasmSafeClient} from './interfaces'; +import '@subql/types-core/dist/global'; declare global { const apiUnsafe: CosmWasmClient; //requires --unsafe flag to be defined From 53ff7d95e2b602801c3f29530ecefe14cc63516a Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Tue, 31 Oct 2023 18:14:47 +1300 Subject: [PATCH 3/3] Update changelog --- packages/common-cosmos/CHANGELOG.md | 2 ++ packages/node/CHANGELOG.md | 2 ++ packages/types/CHANGELOG.md | 2 ++ 3 files changed, 6 insertions(+) diff --git a/packages/common-cosmos/CHANGELOG.md b/packages/common-cosmos/CHANGELOG.md index 83675512a..b71a32a55 100644 --- a/packages/common-cosmos/CHANGELOG.md +++ b/packages/common-cosmos/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Update `@subql/common` and relevant changes to support endBlock feature (#195) ## [3.1.1] - 2023-10-26 ### Changed diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index d0a65cb33..2a0dbb8e4 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Update `@subql/node-core` with fixes and support for endBlock feature (#195) ### Fixed - Missing dependencies for testing command (#194) - Events getting processed multiple times (#194) diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md index 263991330..a1ae90d20 100644 --- a/packages/types/CHANGELOG.md +++ b/packages/types/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Import `@subql/types-core` global into global so its no longer needed to update tsconfig in projects (#195) ### Added - JSDoc to field explaining definition (#194)