diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 3dc5a8aff..69eba0bdf 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Updates to match changes in `@subql/node-core` (#209) + - Dictionary service to use dictionary registry + - Use yargs from node core ## [3.3.1] - 2023-11-08 ### Fixed diff --git a/packages/node/package.json b/packages/node/package.json index 7f01d545f..60f3e5bd4 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -30,7 +30,7 @@ "@nestjs/schedule": "^3.0.1", "@subql/common": "^3.3.0", "@subql/common-cosmos": "workspace:*", - "@subql/node-core": "^6.3.0", + "@subql/node-core": "^6.4.0", "@subql/types-cosmos": "workspace:*", "cosmjs-types": "^0.7.0", "cron-converter": "^1.0.2", diff --git a/packages/node/src/indexer/dictionary.service.spec.ts b/packages/node/src/indexer/dictionary.service.spec.ts index 4c97ed9d4..8abe5ea01 100644 --- a/packages/node/src/indexer/dictionary.service.spec.ts +++ b/packages/node/src/indexer/dictionary.service.spec.ts @@ -8,8 +8,8 @@ import { DictionaryService } from './dictionary.service'; describe('dictionary service', () => { let dictionaryService: DictionaryService; - beforeEach(() => { - dictionaryService = new DictionaryService( + beforeEach(async () => { + dictionaryService = await DictionaryService.create( { network: { chainId: 'juno-1', diff --git a/packages/node/src/indexer/dictionary.service.ts b/packages/node/src/indexer/dictionary.service.ts index ae85cf231..2b65ec24a 100644 --- a/packages/node/src/indexer/dictionary.service.ts +++ b/packages/node/src/indexer/dictionary.service.ts @@ -3,6 +3,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; +import { NETWORK_FAMILY } from '@subql/common'; import { NodeConfig, DictionaryService as CoreDictionaryService, @@ -11,17 +12,34 @@ import { SubqueryProject } from '../configure/SubqueryProject'; @Injectable() export class DictionaryService extends CoreDictionaryService { - constructor( + private constructor( @Inject('ISubqueryProject') protected project: SubqueryProject, nodeConfig: NodeConfig, eventEmitter: EventEmitter2, + dictionaryUrl?: string, ) { super( - project.network.dictionary, + dictionaryUrl ?? project.network.dictionary, project.network.chainId, nodeConfig, eventEmitter, ['lastProcessedHeight', 'chain'], ); } + + static async create( + project: SubqueryProject, + nodeConfig: NodeConfig, + eventEmitter: EventEmitter2, + ): Promise { + const url = + project.network.dictionary ?? + (await CoreDictionaryService.resolveDictionary( + NETWORK_FAMILY.cosmos, + project.network.chainId, + nodeConfig.dictionaryRegistry, + )); + + return new DictionaryService(project, nodeConfig, eventEmitter, url); + } } diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index 3a9d80c45..869815050 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -123,7 +123,22 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; ConnectionPoolService, IndexingBenchmarkService, PoiBenchmarkService, - DictionaryService, + { + provide: DictionaryService, + useFactory: async ( + project: SubqueryProject, + nodeConfig: NodeConfig, + eventEmitter: EventEmitter2, + ) => { + const dictionaryService = await DictionaryService.create( + project, + nodeConfig, + eventEmitter, + ); + return dictionaryService; + }, + inject: ['ISubqueryProject', NodeConfig, EventEmitter2], + }, SandboxService, DsProcessorService, DynamicDsService, diff --git a/packages/node/src/indexer/fetch.service.spec.ts b/packages/node/src/indexer/fetch.service.spec.ts index 83ec9fa9b..5c05edf3d 100644 --- a/packages/node/src/indexer/fetch.service.spec.ts +++ b/packages/node/src/indexer/fetch.service.spec.ts @@ -19,8 +19,11 @@ const nodeConfig = new NodeConfig({ dictionaryTimeout: 10, }); -function mockDictionaryService(url: string): DictionaryServicePrivate { - return new DictionaryService( +// eslint-disable-next-line @typescript-eslint/require-await +async function mockDictionaryService( + url: string, +): Promise { + return DictionaryService.create( { network: { dictionary: url, @@ -34,8 +37,8 @@ function mockDictionaryService(url: string): DictionaryServicePrivate { describe('Dictionary Queries', () => { let dictionary: DictionaryServicePrivate; - beforeAll(() => { - dictionary = mockDictionaryService('http://localhost:3000'); // TODO get url + beforeAll(async () => { + dictionary = await mockDictionaryService('http://localhost:3000'); // TODO get url }); describe('Message Filter Queries', () => { diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts index edc5d7a0d..978a67dbc 100644 --- a/packages/node/src/yargs.ts +++ b/packages/node/src/yargs.ts @@ -1,286 +1,25 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { initLogger } from '@subql/node-core/logger'; -import { hideBin } from 'yargs/helpers'; -import yargs from 'yargs/yargs'; +import { yargsBuilder } from '@subql/node-core/yargs'; -export const yargsOptions = yargs(hideBin(process.argv)) - .env('SUBQL_NODE') - .command({ - command: 'test', - describe: 'Run tests for a SubQuery application', - builder: {}, - handler: (argv) => { - initLogger( - argv.debug as string, - argv.outputFmt as 'json' | 'colored', - argv.logLevel as string | undefined, - ); - // lazy import to make sure logger is instantiated before all other services - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { testingInit } = require('./subcommands/testing.init'); - return testingInit(); - }, - }) - .command({ - command: 'force-clean', - describe: - 'Clean the database dropping project schemas and tables. Once the command is executed, the application would exit upon completion.', - builder: {}, - handler: (argv) => { - initLogger( - argv.debug as string, - argv.outputFmt as 'json' | 'colored', - argv.logLevel as string | undefined, - ); - - // lazy import to make sure logger is instantiated before all other services - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { forceCleanInit } = require('./subcommands/forceClean.init'); - return forceCleanInit(); - }, - }) - .command({ - command: 'reindex', - describe: - 'Reindex to specified block height. Historical must be enabled for the targeted project (--disable-historical=false). Once the command is executed, the application would exit upon completion.', - builder: (yargs) => - yargs.options('targetHeight', { - type: 'number', - description: 'set targetHeight', - require: true, - }), - handler: (argv) => { - initLogger( - argv.debug as string, - argv.outputFmt as 'json' | 'colored', - argv.logLevel as string | undefined, - ); - // lazy import to make sure logger is instantiated before all other services - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { reindexInit } = require('./subcommands/reindex.init'); - return reindexInit(argv.targetHeight); - }, - }) - .options({ - 'batch-size': { - demandOption: false, - describe: 'Batch size of blocks to fetch in one round', - type: 'number', - }, - config: { - alias: 'c', - demandOption: false, - describe: 'Specify configuration file', - type: 'string', - }, - 'db-schema': { - demandOption: false, - describe: 'Db schema name of the project', - type: 'string', - }, - debug: { - demandOption: false, - describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary". To disable specific scopes you can prefix them with '-'. e.g. "*,-SQL"`, - type: 'string', - }, - 'dictionary-resolver': { - demandOption: false, - describe: 'Use SubQuery Network dictionary resolver', - type: 'string', - }, - 'dictionary-timeout': { - demandOption: false, - describe: 'Max timeout for dictionary query', - type: 'number', - }, - 'dictionary-query-size': { - demandOption: false, - describe: - 'Dictionary query max block size, this specify the block height range of the dictionary query', - type: 'number', - }, - 'disable-historical': { - demandOption: false, - describe: 'Disable storing historical state entities', - type: 'boolean', - // NOTE: don't set a default for this. It will break apply args from manifest. The default should be set in NodeConfig - }, - ipfs: { - demandOption: false, - describe: 'IPFS gateway endpoint', - type: 'string', - }, - local: { - deprecated: true, - type: 'boolean', - demandOption: false, - describe: 'Use local mode', - }, - 'log-level': { - demandOption: false, - describe: 'Specify log level to print. Ignored when --debug is used', - type: 'string', - choices: ['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent'], - }, - 'multi-chain': { - demandOption: false, - default: false, - describe: - 'Enables indexing multiple subquery projects into the same database schema', - type: 'boolean', - }, - 'network-dictionary': { - alias: 'd', - demandOption: false, - describe: 'Specify the dictionary api for this network', - type: 'string', - }, - 'network-endpoint': { - demandOption: false, - type: 'string', - describe: 'Blockchain network endpoint to connect', - }, - 'output-fmt': { - demandOption: false, - describe: 'Print log as json or plain text', - type: 'string', - choices: ['json', 'colored'], - }, - port: { - alias: 'p', - demandOption: false, - describe: 'The port the service will bind to', - type: 'number', - }, - profiler: { - demandOption: false, - describe: 'Show profiler information to console output', - type: 'boolean', - default: false, - }, - 'proof-of-index': { - demandOption: false, - describe: 'Enable/disable proof of index', - type: 'boolean', - default: false, - }, - 'query-limit': { - demandOption: false, - describe: - 'The limit of items a project can query with store.getByField at once', - type: 'number', - default: 100, - }, - 'scale-batch-size': { - type: 'boolean', - demandOption: false, - describe: 'scale batch size based on memory usage', - default: false, - }, - 'pg-ca': { - demandOption: false, - describe: - 'Postgres ca certificate - to enable TLS/SSL connections to your PostgreSQL, path to the server certificate file are required, e.g /path/to/server-certificates/root.crt', - type: 'string', - }, - 'pg-key': { - demandOption: false, - describe: - 'Postgres client key - Path to key file e.g /path/to/client-key/postgresql.key', - type: 'string', - }, - 'pg-cert': { - demandOption: false, - describe: - 'Postgres client certificate - Path to client certificate e.g /path/to/client-certificates/postgresql.crt', - type: 'string', - }, - 'store-cache-threshold': { - demandOption: false, - describe: - 'Store cache will flush data to the database when number of records excess this threshold', - type: 'number', - }, - 'store-cache-upper-limit': { - demandOption: false, - describe: - 'Defines the upper limit to the store cache size. When this limit is reached indexing will wait for the cache to be flushed before continuing.', - type: 'number', - }, - 'store-get-cache-size': { - demandOption: false, - describe: 'Store get cache size for each model', - type: 'number', - }, - 'store-cache-async': { - demandOption: false, - describe: - 'If enabled the store cache will flush data asyncronously relative to indexing data', - type: 'boolean', - }, - 'store-flush-interval': { - demandOption: false, - describe: - 'The interval, in seconds, at which data is flushed from the cache. ' + - 'This ensures that data is persisted regularly when there is either not much data or the project is up to date.', - type: 'number', - default: 5, - }, - subquery: { - alias: 'f', - demandOption: true, - default: process.cwd(), - describe: 'Local path or IPFS cid of the subquery project', - type: 'string', - }, - 'subquery-name': { - deprecated: true, - demandOption: false, - describe: 'Name of the subquery project', - type: 'string', - }, - subscription: { - demandOption: false, - describe: 'Enable subscription by create notification triggers', - type: 'boolean', - default: false, - }, - timeout: { - demandOption: false, - describe: 'Timeout for indexer sandbox to execute the mapping functions', - type: 'number', - }, - 'timestamp-field': { - demandOption: false, - describe: 'Enable/disable created_at and updated_at in schema', - type: 'boolean', - default: false, - }, - 'unfinalized-blocks': { - demandOption: false, - describe: 'Enable to fetch and index unfinalized blocks', - type: 'boolean', - // NOTE: don't set a default for this. It will break apply args from manifest. The default should be set in NodeConfig - }, - unsafe: { - type: 'boolean', - demandOption: false, - describe: 'Allows usage of any built-in module within the sandbox', - // NOTE: don't set a default for this. It will break apply args from manifest. The default should be set in NodeConfig - }, - workers: { - alias: 'w', - demandOption: false, - describe: - 'Number of worker threads to use for fetching and processing blocks. Disabled by default.', - type: 'number', - }, - root: { - describe: - 'This is a hidden flag only used from the main thread to workers. It provides a root directory for the project. This is a temp directory with IPFS and GitHub projects.', - type: 'string', - }, - }) - .hide('root'); // root is hidden because its for internal use +export const yargsOptions = yargsBuilder({ + initTesting: () => { + // lazy import to make sure logger is instantiated before all other services + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { testingInit } = require('./subcommands/testing.init'); + return testingInit(); + }, + initForceClean: () => { + // lazy import to make sure logger is instantiated before all other services + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { forceCleanInit } = require('./subcommands/forceClean.init'); + return forceCleanInit(); + }, + initReindex: (targetHeight: number) => { + // lazy import to make sure logger is instantiated before all other services + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { reindexInit } = require('./subcommands/reindex.init'); + return reindexInit(targetHeight); + }, +}); diff --git a/yarn.lock b/yarn.lock index 120fa7cec..4a94cb90e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3794,7 +3794,7 @@ __metadata: languageName: node linkType: hard -"@polkadot/util-crypto@npm:^12.2.1": +"@polkadot/util-crypto@npm:^12.5.1": version: 12.5.1 resolution: "@polkadot/util-crypto@npm:12.5.1" dependencies: @@ -3814,7 +3814,7 @@ __metadata: languageName: node linkType: hard -"@polkadot/util@npm:12.5.1, @polkadot/util@npm:^12.2.1": +"@polkadot/util@npm:12.5.1, @polkadot/util@npm:^12.5.1": version: 12.5.1 resolution: "@polkadot/util@npm:12.5.1" dependencies: @@ -4291,9 +4291,9 @@ __metadata: languageName: node linkType: hard -"@subql/node-core@npm:^6.3.0": - version: 6.3.0 - resolution: "@subql/node-core@npm:6.3.0" +"@subql/node-core@npm:^6.4.0": + version: 6.4.0 + resolution: "@subql/node-core@npm:6.4.0" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 @@ -4303,7 +4303,7 @@ __metadata: "@subql/common": 3.3.0 "@subql/testing": 2.1.0 "@subql/types": 3.3.0 - "@subql/utils": 2.5.0 + "@subql/utils": 2.6.0 "@subql/x-sequelize": 6.32.0-0.0.2 "@willsoto/nestjs-prometheus": ^5.4.0 async-lock: ^1.4.0 @@ -4318,7 +4318,7 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: a0961eb85a5bfb5ff521658563ef78e2db55969857295f4108a937377fbf81bb02567d8353be24b880e38e9b6c14d91ebad468fe555346faae7d8a0a5ae69f79 + checksum: 29208b471b1c41750905b5056fbaab16c88f5448ca4d525ccdc5156b513fe66debf9424519dbbf277ce47fe25e0cdde6186c12fdec233a8aedc1b269b56f8fde languageName: node linkType: hard @@ -4339,7 +4339,7 @@ __metadata: "@nestjs/testing": ^9.4.0 "@subql/common": ^3.3.0 "@subql/common-cosmos": "workspace:*" - "@subql/node-core": ^6.3.0 + "@subql/node-core": ^6.4.0 "@subql/types-cosmos": "workspace:*" "@types/express": ^4.17.13 "@types/jest": ^27.4.0 @@ -4414,12 +4414,12 @@ __metadata: languageName: node linkType: hard -"@subql/utils@npm:2.5.0": - version: 2.5.0 - resolution: "@subql/utils@npm:2.5.0" +"@subql/utils@npm:2.6.0": + version: 2.6.0 + resolution: "@subql/utils@npm:2.6.0" dependencies: - "@polkadot/util": ^12.2.1 - "@polkadot/util-crypto": ^12.2.1 + "@polkadot/util": ^12.5.1 + "@polkadot/util-crypto": ^12.5.1 "@subql/x-sequelize": 6.32.0-0.0.2 ansi-styles: ^6.1.0 axios: ^0.27.1 @@ -4433,7 +4433,7 @@ __metadata: rotating-file-stream: ^3.0.2 semver: ^7.5.2 tar: ^6.1.11 - checksum: 27d958f916f519aba9257d5bb4d310f0bb08d628098643c417342fbf864e6138de28d7fd0c8a3b3177f93668544b0655a6b483c9b1fe5fabbd8b08babc2ea36a + checksum: 284b125beb62976bd6c5f49a82d8bd3da96f928bfbfde72f42773b62440c6b9ef1a68dfbae85cff7aee40f5f349ebb1a91f41a314cbfd2b98c905496543a1323 languageName: node linkType: hard