From 8138f4e858d5bcdd50068906e9d7cc9d6a31953b Mon Sep 17 00:00:00 2001 From: JQQQ Date: Fri, 20 Oct 2023 14:19:57 +1300 Subject: [PATCH] [release] 20231020 --- packages/common-cosmos/CHANGELOG.md | 7 +- packages/common-cosmos/package.json | 4 +- packages/node/CHANGELOG.md | 10 ++- packages/node/package.json | 6 +- .../worker-block-dispatcher.service.ts | 8 +- packages/node/src/indexer/fetch.module.ts | 5 ++ packages/node/src/indexer/sandbox.service.ts | 4 + .../src/indexer/worker/worker-fetch.module.ts | 6 ++ .../worker/worker.inMemoryCache.service.ts | 20 +++++ packages/types/CHANGELOG.md | 7 +- packages/types/package.json | 4 +- yarn.lock | 77 +++++++++++-------- 12 files changed, 113 insertions(+), 45 deletions(-) create mode 100644 packages/node/src/indexer/worker/worker.inMemoryCache.service.ts diff --git a/packages/common-cosmos/CHANGELOG.md b/packages/common-cosmos/CHANGELOG.md index 27d0554a0..49f9cb5fb 100644 --- a/packages/common-cosmos/CHANGELOG.md +++ b/packages/common-cosmos/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.1.0] - 2023-10-20 +### Changed +- Version bump with `common` 3.1.3 + ## [3.0.3] - 2023-10-17 ### Changed - Update type names to be consistent with main SDK (#189) @@ -95,7 +99,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix chainTypes not being in deployments ## [0.0.5] - 2022-06-15 -[Unreleased]: https://github.com/subquery/subql-cosmos/compare/common-cosmos/3.0.3...HEAD +[Unreleased]: https://github.com/subquery/subql-cosmos/compare/common-cosmos/3.1.0...HEAD +[3.1.0]: https://github.com/subquery/subql-cosmos/compare/common-cosmos/3.0.3...common-cosmos/3.1.0 [3.0.3]: https://github.com/subquery/subql-cosmos/compare/common-cosmos/3.0.2...common-cosmos/3.0.3 [3.0.2]: https://github.com/subquery/subql-cosmos/compare/common-cosmos/3.0.1...common-cosmos/3.0.2 [3.0.1]: https://github.com/subquery/subql-cosmos/compare/common-cosmos/3.0.0...common-cosmos/3.0.1 diff --git a/packages/common-cosmos/package.json b/packages/common-cosmos/package.json index 6f0b1fc00..2c87ad520 100644 --- a/packages/common-cosmos/package.json +++ b/packages/common-cosmos/package.json @@ -1,6 +1,6 @@ { "name": "@subql/common-cosmos", - "version": "3.0.3", + "version": "3.1.0", "description": "", "scripts": { "build": "rm -rf dist && tsc -b", @@ -23,7 +23,7 @@ "@protobufs/google": "^0.0.10", "@protobufs/ibc": "^0.1.0", "@protobufs/tendermint": "^0.0.10", - "@subql/common": "^3.1.2", + "@subql/common": "^3.1.3", "@subql/types-cosmos": "workspace:*", "fs-extra": "^11.1.1", "js-yaml": "^4.1.0", diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 4a8a88146..4ae2fe24c 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.1.0] - 2023-10-20 +### Added +- Inject in-memory cache to sandbox + +### Fixed +- Bump with `@subq/node-core` 3.1.0 , fixed poi migration init check, and improve logging + ## [3.0.3] - 2023-10-17 ### Changed - Update type names to be consistent with main SDK (#189) @@ -284,7 +291,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Init release -[Unreleased]: https://github.com/subquery/subql-cosmos/compare/node-cosmos/3.0.3...HEAD +[Unreleased]: https://github.com/subquery/subql-cosmos/compare/node-cosmos/3.1.0...HEAD +[3.1.0]: https://github.com/subquery/subql-cosmos/compare/node-cosmos/3.0.3...node-cosmos/3.1.0 [3.0.3]: https://github.com/subquery/subql-cosmos/compare/node-cosmos/3.0.2...node-cosmos/3.0.3 [3.0.2]: https://github.com/subquery/subql-cosmos/compare/node-cosmos/3.0.1...node-cosmos/3.0.2 [3.0.1]: https://github.com/subquery/subql-cosmos/compare/node-cosmos/3.0.0...node-cosmos/3.0.1 diff --git a/packages/node/package.json b/packages/node/package.json index 8f1d70da8..12e3b306e 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -1,6 +1,6 @@ { "name": "@subql/node-cosmos", - "version": "3.0.3", + "version": "3.1.0", "description": "", "author": "SubQuery Pte Ltd", "license": "GPL-3.0", @@ -28,9 +28,9 @@ "@nestjs/event-emitter": "^2.0.0", "@nestjs/platform-express": "^9.4.0", "@nestjs/schedule": "^3.0.1", - "@subql/common": "^3.1.2", + "@subql/common": "^3.1.3", "@subql/common-cosmos": "workspace:*", - "@subql/node-core": "^6.0.2", + "@subql/node-core": "^6.1.0", "@subql/types-cosmos": "workspace:*", "cosmjs-types": "^0.7.0", "cron-converter": "^1.0.2", diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index 10b827f51..646caac38 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -24,10 +24,12 @@ import { HostUnfinalizedBlocks, baseWorkerFunctions, storeHostFunctions, + cacheHostFunctions, dynamicDsHostFunctions, IProjectUpgradeService, + InMemoryCacheService, } from '@subql/node-core'; -import { Store } from '@subql/types-core'; +import { Cache, Store } from '@subql/types-core'; import { CosmosProjectDs, SubqueryProject, @@ -44,6 +46,7 @@ type IndexerWorker = IIndexerWorker & { async function createIndexerWorker( store: Store, + cache: Cache, dynamicDsService: IDynamicDsService, unfinalizedBlocksService: IUnfinalizedBlocksService, connectionPoolState: ConnectionPoolStateManager, @@ -60,6 +63,7 @@ async function createIndexerWorker( path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'), [...baseWorkerFunctions, 'initWorker'], { + ...cacheHostFunctions(cache), ...storeHostFunctions(store), ...dynamicDsHostFunctions(dynamicDsService), unfinalizedBlocksProcess: @@ -88,6 +92,7 @@ export class WorkerBlockDispatcherService @Inject('IProjectUpgradeService') projectUpgadeService: IProjectUpgradeService, smartBatchService: SmartBatchService, + cacheService: InMemoryCacheService, storeService: StoreService, storeCacheService: StoreCacheService, poiService: PoiService, @@ -112,6 +117,7 @@ export class WorkerBlockDispatcherService () => createIndexerWorker( storeService.getStore(), + cacheService.getCache(), dynamicDsService, unfinalizedBlocksSevice, connectionPoolState, diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index 74e30d496..3a9d80c45 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -15,6 +15,7 @@ import { StoreCacheService, ConnectionPoolStateManager, IProjectUpgradeService, + InMemoryCacheService, } from '@subql/node-core'; import { SubqueryProject } from '../configure/SubqueryProject'; import { ApiService } from './api.service'; @@ -34,6 +35,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; @Module({ providers: [ + InMemoryCacheService, StoreService, StoreCacheService, ApiService, @@ -56,6 +58,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; apiService: ApiService, indexerManager: IndexerManager, smartBatchService: SmartBatchService, + cacheService: InMemoryCacheService, storeService: StoreService, storeCacheService: StoreCacheService, poiService: PoiService, @@ -72,6 +75,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; projectService, projectUpgradeService, smartBatchService, + cacheService, storeService, storeCacheService, poiService, @@ -104,6 +108,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; ApiService, IndexerManager, SmartBatchService, + InMemoryCacheService, StoreService, StoreCacheService, PoiService, diff --git a/packages/node/src/indexer/sandbox.service.ts b/packages/node/src/indexer/sandbox.service.ts index f57ed905f..6e8f95f9c 100644 --- a/packages/node/src/indexer/sandbox.service.ts +++ b/packages/node/src/indexer/sandbox.service.ts @@ -9,6 +9,7 @@ import { IndexerSandbox, hostStoreToStore, ISubqueryProject, + InMemoryCacheService, } from '@subql/node-core'; import { Store, BaseDataSource } from '@subql/types-core'; import { ApiService, CosmosSafeClient } from './api.service'; @@ -22,6 +23,7 @@ export class SandboxService { private readonly apiService: ApiService, @Inject(isMainThread ? StoreService : 'Null') private readonly storeService: StoreService, + private readonly cacheService: InMemoryCacheService, private readonly nodeConfig: NodeConfig, @Inject('ISubqueryProject') private readonly project: ISubqueryProject, ) {} @@ -31,11 +33,13 @@ export class SandboxService { ? this.storeService.getStore() : hostStoreToStore((global as any).host); // Provided in worker.ts + const cache = this.cacheService.getCache(); const entry = this.getDataSourceEntry(ds); let processor = this.processorCache[entry]; if (!processor) { processor = new IndexerSandbox( { + cache, store, root: this.project.root, entry, diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts index db1eb7e11..4d394da70 100644 --- a/packages/node/src/indexer/worker/worker-fetch.module.ts +++ b/packages/node/src/indexer/worker/worker-fetch.module.ts @@ -8,6 +8,7 @@ import { WorkerDynamicDsService, ConnectionPoolStateManager, WorkerConnectionPoolStateManager, + InMemoryCacheService, } from '@subql/node-core'; import { SubqueryProject } from '../../configure/SubqueryProject'; import { ApiService } from '../api.service'; @@ -18,6 +19,7 @@ import { IndexerManager } from '../indexer.manager'; import { ProjectService } from '../project.service'; import { SandboxService } from '../sandbox.service'; import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; +import { WorkerInMemoryCacheService } from './worker.inMemoryCache.service'; import { WorkerService } from './worker.service'; import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.service'; @@ -63,6 +65,10 @@ import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.servi useFactory: () => new WorkerUnfinalizedBlocksService((global as any).host), }, + { + provide: InMemoryCacheService, + useFactory: () => new WorkerInMemoryCacheService((global as any).host), + }, ], exports: [], }) diff --git a/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts b/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts new file mode 100644 index 000000000..1011f9545 --- /dev/null +++ b/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts @@ -0,0 +1,20 @@ +// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import { isMainThread } from 'worker_threads'; +import { Injectable } from '@nestjs/common'; +import { HostCache, hostCacheToCache } from '@subql/node-core'; +import { Cache } from '@subql/types-core'; + +@Injectable() +export class WorkerInMemoryCacheService { + constructor(private host: HostCache) { + if (isMainThread) { + throw new Error('Expected to be worker thread'); + } + } + + getCache(): Cache { + return hostCacheToCache(this.host); + } +} diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md index bf2ea5a85..5d0b9df47 100644 --- a/packages/types/CHANGELOG.md +++ b/packages/types/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.1.0] - 2023-10-20 +### Changed +- Version bump with `types-core` 0.2.0 + ## [3.0.3] - 2023-10-17 ### Changed - Update type names to be consistent with main SDK (#189) @@ -81,7 +85,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix chainTypes not being in deployments ## [0.0.5] - 2022-06-15 -[Unreleased]: https://github.com/subquery/subql-cosmos/compare/types-cosmos/3.0.3...HEAD +[Unreleased]: https://github.com/subquery/subql-cosmos/compare/types-cosmos/3.1.0...HEAD +[3.1.0]: https://github.com/subquery/subql-cosmos/compare/types-cosmos/3.0.3...types-cosmos/3.1.0 [3.0.3]: https://github.com/subquery/subql-cosmos/compare/types-cosmos/3.0.2...types-cosmos/3.0.3 [3.0.2]: https://github.com/subquery/subql-cosmos/compare/types-cosmos/3.0.1...types-cosmos/3.0.2 [3.0.1]: https://github.com/subquery/subql-cosmos/compare/types-cosmos/3.0.0...types-cosmos/3.0.1 diff --git a/packages/types/package.json b/packages/types/package.json index 2c1b222f4..1267e2da7 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@subql/types-cosmos", - "version": "3.0.3", + "version": "3.1.0", "description": "", "homepage": "https://github.com/subquery/subql-cosmos", "repository": "github:subquery/subql-cosmos", @@ -19,6 +19,6 @@ "@cosmjs/cosmwasm-stargate": "^0.30.1", "@cosmjs/proto-signing": "^0.30.1", "@cosmjs/stargate": "^0.30.1", - "@subql/types-core": "^0.1.1" + "@subql/types-core": "^0.2.0" } } diff --git a/yarn.lock b/yarn.lock index e88309bf0..323cadaec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4251,7 +4251,7 @@ __metadata: "@protobufs/google": ^0.0.10 "@protobufs/ibc": ^0.1.0 "@protobufs/tendermint": ^0.0.10 - "@subql/common": ^3.1.2 + "@subql/common": ^3.1.3 "@subql/types-cosmos": "workspace:*" "@types/bn.js": 4.11.6 "@types/ejs": ^3.1.2 @@ -4273,11 +4273,11 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:3.1.2, @subql/common@npm:^3.1.2": - version: 3.1.2 - resolution: "@subql/common@npm:3.1.2" +"@subql/common@npm:3.1.3, @subql/common@npm:^3.1.3": + version: 3.1.3 + resolution: "@subql/common@npm:3.1.3" dependencies: - "@subql/types-core": 0.1.1 + "@subql/types-core": 0.1.2-0 axios: ^0.27.2 class-transformer: ^0.5.1 class-validator: ^0.14.0 @@ -4287,23 +4287,23 @@ __metadata: reflect-metadata: ^0.1.13 semver: ^7.5.2 update-notifier: 5.1.0 - checksum: e979fac54089276de67a3a498bdf53396ac3726542d9f38350bced06f45921ab937a885a9c6cf183f3f5e7c82877c8ce9e27a5a5bf1f6fae6113baf68b4b3e55 + checksum: 2e5f145a44913dda9d8cd5d9cfe1bbda6be8e46ef74b278191f67310ffbfeea6ce76c4ff79c5b6b4a9e21a93d60994bc31db29e7bd198cdafe4203723b59893c languageName: node linkType: hard -"@subql/node-core@npm:^6.0.2": - version: 6.0.2 - resolution: "@subql/node-core@npm:6.0.2" +"@subql/node-core@npm:^6.1.0": + version: 6.1.0 + resolution: "@subql/node-core@npm:6.1.0" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 "@subql/apollo-links": ^1.0.2 - "@subql/common": 3.1.2 - "@subql/testing": 2.0.3-0 - "@subql/types": 3.1.3-0 - "@subql/utils": 2.4.4 + "@subql/common": 3.1.3 + "@subql/testing": 2.1.0 + "@subql/types": 3.2.0 + "@subql/utils": 2.4.5-0 "@subql/x-sequelize": 6.32.0-0.0.2 "@willsoto/nestjs-prometheus": ^5.4.0 async-lock: ^1.4.0 @@ -4318,7 +4318,7 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: ff07ba7e9abadcf0ddc9be3672fd4059341a28e1385291cb6bdfb92a3a517796245ee92d421b3d11f533465a3674fab1d32e7c3763ae97144fe034e7d5ecd5a3 + checksum: 935abc9b53644eb7f2b87ae93764e07e03c3bfee00f90b922fc33224ce76c720d163b30d36b3f71f25ee62d085287b4df361d43744c4e682dc7a3ef29032343e languageName: node linkType: hard @@ -4337,9 +4337,9 @@ __metadata: "@nestjs/schedule": ^3.0.1 "@nestjs/schematics": ^9.2.0 "@nestjs/testing": ^9.4.0 - "@subql/common": ^3.1.2 + "@subql/common": ^3.1.3 "@subql/common-cosmos": "workspace:*" - "@subql/node-core": ^6.0.2 + "@subql/node-core": ^6.1.0 "@subql/types-cosmos": "workspace:*" "@types/express": ^4.17.13 "@types/jest": ^27.4.0 @@ -4367,30 +4367,39 @@ __metadata: languageName: unknown linkType: soft -"@subql/testing@npm:2.0.3-0": - version: 2.0.3-0 - resolution: "@subql/testing@npm:2.0.3-0" +"@subql/testing@npm:2.1.0": + version: 2.1.0 + resolution: "@subql/testing@npm:2.1.0" dependencies: - "@subql/types-core": ^0.0.0-1 - checksum: ded64670e266309115f1f8003dbf0cef1219ceff7fa30c2f1389588d1e659685b74e44e70d3f3389654a0e2ba2d04dddb239b19253d35af494cc8a93548955b5 + "@subql/types-core": ^0.1.2-0 + checksum: 42885519277ec48d079accbc68ca600da64e1c960930d506dcd4a207188ad399ff42f81dfeb5d447f235f3bf7ef61ea55d332d47cb10e6d89c4fafda61fe8bac languageName: node linkType: hard -"@subql/types-core@npm:0.1.1, @subql/types-core@npm:^0.1.1": - version: 0.1.1 - resolution: "@subql/types-core@npm:0.1.1" +"@subql/types-core@npm:0.1.2-0": + version: 0.1.2-0 + resolution: "@subql/types-core@npm:0.1.2-0" dependencies: package-json-type: ^1.0.3 - checksum: 6a88547e5091795d2f9f24b5373ce531066bc04602b18f05cad77c5d1953523be479801447cd85e4f03468997e988b8c853fdb409631f8a396525cc723978afa + checksum: a70258c1cbe8aa407946ed4cd8fa13c6fea72caa0efa1837a4c400c535b0e4d483388d8c54eedd1afa483e65ac85706d637e0112230129e9313a90a552d7244f languageName: node linkType: hard -"@subql/types-core@npm:^0.0.0-1": - version: 0.0.0-0-ts-manifest - resolution: "@subql/types-core@npm:0.0.0-0-ts-manifest" +"@subql/types-core@npm:^0.1.2-0": + version: 0.1.2-1 + resolution: "@subql/types-core@npm:0.1.2-1" + dependencies: + package-json-type: ^1.0.3 + checksum: 1c54fecfd7b99875528a87109a3cb8cfcb86a61221f7e715be49198381c6c97e481b60f351c8210df67d1074d3541cd6ee0aad17831f0896dcea63e9784bda64 + languageName: node + linkType: hard + +"@subql/types-core@npm:^0.2.0": + version: 0.2.0 + resolution: "@subql/types-core@npm:0.2.0" dependencies: package-json-type: ^1.0.3 - checksum: e37e0df66471d763200baa00cf2bea526cdf0a5ee81b496473675e5404fe9553d56a6f7ebb6a6a22916981f6a2a2e2d303c538d8d3120fcd81267cc4703a88d2 + checksum: ef538b84cb0c32f41269fdb6708e9e0657da9e7f0469ef24882e0b392a797eaf35364a7ef9368c0c89b799d6522a66cbf17781deae1edfdc8a17f9cd43aed95f languageName: node linkType: hard @@ -4401,7 +4410,7 @@ __metadata: "@cosmjs/cosmwasm-stargate": ^0.30.1 "@cosmjs/proto-signing": ^0.30.1 "@cosmjs/stargate": ^0.30.1 - "@subql/types-core": ^0.1.1 + "@subql/types-core": ^0.2.0 languageName: unknown linkType: soft @@ -4414,9 +4423,9 @@ __metadata: languageName: node linkType: hard -"@subql/utils@npm:2.4.4": - version: 2.4.4 - resolution: "@subql/utils@npm:2.4.4" +"@subql/utils@npm:2.4.5-0": + version: 2.4.5-0 + resolution: "@subql/utils@npm:2.4.5-0" dependencies: "@polkadot/util": ^12.2.1 "@polkadot/util-crypto": ^12.2.1 @@ -4433,7 +4442,7 @@ __metadata: rotating-file-stream: ^3.0.2 semver: ^7.5.2 tar: ^6.1.11 - checksum: 1b0a4a4435bddc2bb058283a4417611be5b8c0248a99dc9f5ea32a1255b60beb64217e49e6bf1c99fef14625e9fa561bdf5a481e3208229093cdca8dfd59a71f + checksum: 7d76203fcb25251225ce7a2a56bf3de8e2a98caed4f1598ca50dea4bf96511bec71b7d4bdbec8648a7b76b654259a94dbdf2c66105961e8aebe780c318ad2ae0 languageName: node linkType: hard