From d6b4e570e1f997c7c1e91f51c74436ff8efa4d14 Mon Sep 17 00:00:00 2001 From: Igor Mahov Date: Fri, 22 Sep 2023 11:36:27 +0300 Subject: [PATCH 1/8] feat: implement dump metrics --- common/config/rush/pnpm-lock.yaml | 48 +++++++++---------- substrate/substrate-dump/package.json | 4 +- substrate/substrate-dump/src/dumper.ts | 15 +++++- substrate/substrate-dump/src/main.ts | 1 + substrate/substrate-dump/src/prometheus.ts | 46 ++++++++++++++++++ .../src/layout.ts | 4 +- 6 files changed, 89 insertions(+), 29 deletions(-) create mode 100644 substrate/substrate-dump/src/prometheus.ts diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 59a596fbc..db7374087 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -332,9 +332,6 @@ dependencies: pg: specifier: ^8.11.0 version: 8.11.0 - prom-client: - specifier: ^14.2.0 - version: 14.2.0 semver: specifier: ^7.5.1 version: 7.5.1 @@ -2434,7 +2431,7 @@ packages: dev: false /bintrees@1.0.2: - resolution: {integrity: sha1-SfiW1uhYpKSZ34XDj7OZua/4QPg=} + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} dev: false /blake2b-wasm@2.4.0: @@ -4924,7 +4921,7 @@ packages: dev: false file:projects/astar-erc20.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-5MywoSkMnDYrVzCUjtNRhCHRQSwbhVbT2nPA3t6o14+jJb+hc6wz2nThRf6OgxV5JWmPLBccN7kw9GtuvV/7fA==, tarball: file:projects/astar-erc20.tgz} + resolution: {integrity: sha512-0GUqXR08VgGnsoYgFHbwiMRGO93jLwcFsMCqoQS0fs+zS9NIfzmzrhRorK5Vj9eziJ4Fqx1XEQOqatwaUBuGcw==, tarball: file:projects/astar-erc20.tgz} id: file:projects/astar-erc20.tgz name: '@rush-temp/astar-erc20' version: 0.0.0 @@ -4959,7 +4956,7 @@ packages: dev: false file:projects/balances.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-swswZ5RQXL64nl+TGoJWmOrg8ShRmjGgKq7tHQHccQw8XSI9RabB/Mjmk64EuMj+XnnFOCmBuOFsLm109R/iHw==, tarball: file:projects/balances.tgz} + resolution: {integrity: sha512-gAc85gJrUgMw1sym9WVkqcvtZONZ+SJXN10zhklRSoNjsP30WmfiRDyQlXfYKCYExvG8BnMlC/QJ7IXY5k9JFQ==, tarball: file:projects/balances.tgz} id: file:projects/balances.tgz name: '@rush-temp/balances' version: 0.0.0 @@ -5022,7 +5019,7 @@ packages: dev: false file:projects/data-test.tgz: - resolution: {integrity: sha512-bq9WEaopulEnZWevdBxC6LrpKrA7SIWoOPZSu0DUmhfy3J2oHY08zpziaW3cgROiq8ASMJxxnplIQ0aSL3/JXg==, tarball: file:projects/data-test.tgz} + resolution: {integrity: sha512-FNudAbXMX/nL0eqDKENkSFNUUmyjSeJ8Xh5hOHuoVMT2fdRRdCMIYbGr4Al1rh1uM0ehDAC1aK2YCL7S/ndqbA==, tarball: file:projects/data-test.tgz} name: '@rush-temp/data-test' version: 0.0.0 dependencies: @@ -5038,7 +5035,7 @@ packages: dev: false file:projects/eth-usdc-transfers.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-Ip4E2tgnjxevi9Jjzb7PV0GUEgQWnPnpuZhVTfZv41jCltrC+hfxikYA0lQKz8/2zSvQcRdSC+wKgyr99oPRNQ==, tarball: file:projects/eth-usdc-transfers.tgz} + resolution: {integrity: sha512-ssvLkcogjLcQn2SFp8TjV1jfirgqHS3pUy/uBD+UOgmBl4UxNOfOpxy6eRXsxaN5aRoVvAPNJcHk3raZ6YozuA==, tarball: file:projects/eth-usdc-transfers.tgz} id: file:projects/eth-usdc-transfers.tgz name: '@rush-temp/eth-usdc-transfers' version: 0.0.0 @@ -5072,7 +5069,7 @@ packages: dev: false file:projects/evm-processor.tgz: - resolution: {integrity: sha512-3wB46Cwqb9Yz35aLaFRSbHhoJxMOTfj0E7NumqNT/0aHhk6w3swair5BvUFV9Ro+QJl8uyB3YvbOXfhRP6PDVQ==, tarball: file:projects/evm-processor.tgz} + resolution: {integrity: sha512-F3SVUFtR5BhLgMGTAZHj0P/K4VvWelo2KDZ8gLeKI9H7d2W4401MPbeuEQVvO1UfTYlGWL73YfLyjIELNbIYlw==, tarball: file:projects/evm-processor.tgz} name: '@rush-temp/evm-processor' version: 0.0.0 dependencies: @@ -5081,7 +5078,7 @@ packages: dev: false file:projects/evm-typegen.tgz: - resolution: {integrity: sha512-1mUSI39LkTFudg9I4Ec0aKS1NciBhvl2GBDQ6xjOGs7A06fjAoSsRSMRZ4+w7pfXCNNr8KZTPtdsB7WrcYtObA==, tarball: file:projects/evm-typegen.tgz} + resolution: {integrity: sha512-2BusN9BfyLWgyvWceasHgKiC7EGfW8/UlDnANx5njuq73KxhkbAgTlinSLkwDSD1ovPFRiCafafRa0CT5uPGHg==, tarball: file:projects/evm-typegen.tgz} name: '@rush-temp/evm-typegen' version: 0.0.0 dependencies: @@ -5112,7 +5109,7 @@ packages: dev: false file:projects/gql-test-client.tgz(graphql@15.8.0): - resolution: {integrity: sha512-FviPmrYiW+k4yme1uGKTucqg9mI15WutY1aY1pb87bZZRjdSarPke5C673zn7KD0EhPgMKJKNiB4RNcwFL+/fQ==, tarball: file:projects/gql-test-client.tgz} + resolution: {integrity: sha512-rTQ0oTL6CE+fy4LLaZUANqER3lOFbTmgykPrfptvIH9Ld/uSoMXUIYJMkKSp7hr7rmNwLAjSmiEQoiRqa8LWvA==, tarball: file:projects/gql-test-client.tgz} id: file:projects/gql-test-client.tgz name: '@rush-temp/gql-test-client' version: 0.0.0 @@ -5211,7 +5208,7 @@ packages: dev: false file:projects/ink-typegen.tgz: - resolution: {integrity: sha512-sAQyqL1Zzk0klY1COwPQNpai2AOxRUTZ/JlVtLvdK2bim3Ua+iDFjks7Pn34tP8rEZCBKKha5TOJJq4bzjECEw==, tarball: file:projects/ink-typegen.tgz} + resolution: {integrity: sha512-Mddllizme6fiBnOi1hQJfiRNjl8B0ngj9XCriiuiLpLT5cXAEpRqfTt9Gh2jAlCbwm/07LhPe1dXx+CEWTbWnA==, tarball: file:projects/ink-typegen.tgz} name: '@rush-temp/ink-typegen' version: 0.0.0 dependencies: @@ -5271,7 +5268,7 @@ packages: dev: false file:projects/ops-xcm-typegen.tgz: - resolution: {integrity: sha512-2TN3XmQUwV9Rd2vSI5A44J8I5Rl5WVmEErFOHKcQ1ypsc7t7hK6mLrnsCOSq5aL9n4nHtSTEnYqpZwk5m0KfQg==, tarball: file:projects/ops-xcm-typegen.tgz} + resolution: {integrity: sha512-dGZc7v0Px7pEA0TTfB0QEO/8cJ/T+bWDyrUgyrr2WAuAwmAOzf7dGyoa/jYD90HC6RFOPyambnpDY5xK6qSANg==, tarball: file:projects/ops-xcm-typegen.tgz} name: '@rush-temp/ops-xcm-typegen' version: 0.0.0 dependencies: @@ -5281,7 +5278,7 @@ packages: dev: false file:projects/rpc-client.tgz: - resolution: {integrity: sha512-sW5/WrDGObfpLhj4lZnCgK051uCQ/1/snDP0CXTuOUiCWGMmQfNZqdLNaRPOScsMUgjLVGlgdevX1/2fyqUwrA==, tarball: file:projects/rpc-client.tgz} + resolution: {integrity: sha512-Kmv/93g4enMfQ8Ql8cFnY34qT8tttba1l2lRg4GBhplKwB/LjS165KvnKbSbmUO/ju4XMhhGy6vGF+YyEa9OJQ==, tarball: file:projects/rpc-client.tgz} name: '@rush-temp/rpc-client' version: 0.0.0 dependencies: @@ -5310,7 +5307,7 @@ packages: dev: false file:projects/shibuya-psp22.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-F4vpXzcF89m9e++rZg0xq/6a5lyZ+ZJVJ6WUbP0PowoKLKgc/1SAYSpc7iAPhpm86/apWNEPFaaDwetAoPzwPw==, tarball: file:projects/shibuya-psp22.tgz} + resolution: {integrity: sha512-dofngWD751Lh9Q3o9fOZqFpDVBh3kc1J/Tz3gXTG35/kg/bh/6yqPqh/pyw1aoZeJM6DYfJUrDPzGM7BVTKdTQ==, tarball: file:projects/shibuya-psp22.tgz} id: file:projects/shibuya-psp22.tgz name: '@rush-temp/shibuya-psp22' version: 0.0.0 @@ -5363,7 +5360,7 @@ packages: dev: false file:projects/substrate-data-raw.tgz: - resolution: {integrity: sha512-g6iMT8yJ/Pq35Z1l4jKqv0c3rcFgAnyjYlVozJ6Y+wuZCtT7NI4y/8egiQMkZCEBM1buESOXygvhlKyc7h06Fw==, tarball: file:projects/substrate-data-raw.tgz} + resolution: {integrity: sha512-RQXdSGiCi/CIgkM7uejiGrYl8Pn0/tAFdc6GNRvW6H15t3LJBR3/kMLM9Rvbr9OruIUufLwqVllvAO28JpVD7w==, tarball: file:projects/substrate-data-raw.tgz} name: '@rush-temp/substrate-data-raw' version: 0.0.0 dependencies: @@ -5372,7 +5369,7 @@ packages: dev: false file:projects/substrate-data.tgz: - resolution: {integrity: sha512-6i0zC8w5/e+0OwuZ/q/0d6hN21P03ORHOtis3bJvyNHEF6i0dnCDB5KZpVD5RPekboFefp53bYkdiWMJV8wEAw==, tarball: file:projects/substrate-data.tgz} + resolution: {integrity: sha512-uB2QgMbtN6mQbysgVlkwnoNz3//DTiEgcMrUlzLCpNPnY6pEmp3oWyz0x7T5ncP2+5SnEFEQMJwoLspoJLs4SA==, tarball: file:projects/substrate-data.tgz} name: '@rush-temp/substrate-data' version: 0.0.0 dependencies: @@ -5384,17 +5381,18 @@ packages: dev: false file:projects/substrate-dump.tgz: - resolution: {integrity: sha512-K0WH6ekorko2LKs/xjFxaiEpiwtajNRadKFUf0fAaGO7phHa7fS54+UvTkUD+zcJYFn/l7l/dWoQsOvoBA11LQ==, tarball: file:projects/substrate-dump.tgz} + resolution: {integrity: sha512-x6kvkHqxCnKtGOjycMsNiKEMuVBZ5/akZGlPRfSgpGQzVvSrWykF/HxOS/obPomK5wgqNd8koIPtAddy/r8gRg==, tarball: file:projects/substrate-dump.tgz} name: '@rush-temp/substrate-dump' version: 0.0.0 dependencies: '@types/node': 18.16.17 commander: 11.0.0 + prom-client: 14.2.0 typescript: 4.9.5 dev: false file:projects/substrate-ingest.tgz: - resolution: {integrity: sha512-1Y+ZVXm3oBTCbuYaXtBQUXVRdmevP9knXPoXsBRQGxH12N1/+/aAa4pNzSytq9TwCtnusYP5KZO2zrDxkDtDGA==, tarball: file:projects/substrate-ingest.tgz} + resolution: {integrity: sha512-v0XT1a5LHVVrk4YuCJ7jE+a+2wrdWVsePFSR+/hVPNny1raOwstJRgEog4NUrKTcfxSCSpBlqTSR9RLDVByTPA==, tarball: file:projects/substrate-ingest.tgz} name: '@rush-temp/substrate-ingest' version: 0.0.0 dependencies: @@ -5414,7 +5412,7 @@ packages: dev: false file:projects/substrate-metadata-service.tgz: - resolution: {integrity: sha512-oi1L1+t+l1Cba1ELgbUTjnzsgEbKX6GwkNPUnUZF+0WR6RqLM/A90SmM9lqjB7Scix3DBVOWNQDYr3SmSFQGVQ==, tarball: file:projects/substrate-metadata-service.tgz} + resolution: {integrity: sha512-T8LkSti9o/0yzuo/8O10PyDphzwFUyH2ObijiHOheFuOhbNwLK8Go5b/EE/WKMAT0ION9LBcZHUcodDAsy1+DQ==, tarball: file:projects/substrate-metadata-service.tgz} name: '@rush-temp/substrate-metadata-service' version: 0.0.0 dependencies: @@ -5426,7 +5424,7 @@ packages: dev: false file:projects/substrate-processor.tgz: - resolution: {integrity: sha512-Q44B4nSt8ZEhpf7fEnve5aP1iToWueNQbLTfK4Ek9wZFSQo4iNnIa58KvFG2J5Q1O7MGN5qdL41fNVdMWEC5Zw==, tarball: file:projects/substrate-processor.tgz} + resolution: {integrity: sha512-s/E3nzl6nK2Cpjfb/oo+MKYkvtbAQP7Rr+G+kcBL4sXdwbrUyrnQuW2lVq1v3E1xmWrHhPjMw5aDxCpXBIJLPA==, tarball: file:projects/substrate-processor.tgz} name: '@rush-temp/substrate-processor' version: 0.0.0 dependencies: @@ -5450,7 +5448,7 @@ packages: dev: false file:projects/substrate-typegen.tgz: - resolution: {integrity: sha512-rzVCfNklH24EONf2S6YM3L/tueDBTnp90A7amEiXEIU1H7wW8fOG0mq/yyjsGLM3JxPMRkgme64VjbNGrykutw==, tarball: file:projects/substrate-typegen.tgz} + resolution: {integrity: sha512-u1O8Bzc7E5H/QDl+Z5kGfUFPEMPvFpsJyDi7ttIt4aQxm6An/LWG5yyYgkMxb31yF+G3Ks++O29yDLpfEWn+dg==, tarball: file:projects/substrate-typegen.tgz} name: '@rush-temp/substrate-typegen' version: 0.0.0 dependencies: @@ -5567,7 +5565,7 @@ packages: dev: false file:projects/types-test.tgz: - resolution: {integrity: sha512-MjsqYABJmESYkK+o6f9iGwKeXMdN/U4FNIA7JnmIzJdijCrKVTHjHtRogXBM1Uqd6ZeFh+5IWTQMzmjEEkgkAA==, tarball: file:projects/types-test.tgz} + resolution: {integrity: sha512-TipleaCUm6U+nytipQWudVrHLnH1G8HQRhLBTK036DubTMs4TZKgjrHCdlHr7rzPvTy24QzfmVt3Dx8ShNGHfA==, tarball: file:projects/types-test.tgz} name: '@rush-temp/types-test' version: 0.0.0 dependencies: @@ -5576,7 +5574,7 @@ packages: dev: false file:projects/util-internal-archive-client.tgz: - resolution: {integrity: sha512-B+i5lJMxVH1GWc5uIqcmm4K6lTukJw80Vdu+dtoveGwL9AGwrdpBvG4SmMy6MEhAxihqVWJwL56tR7TOYmo6kQ==, tarball: file:projects/util-internal-archive-client.tgz} + resolution: {integrity: sha512-uFEB0GrZXJ47wz1Eqce66X4cppQfsZeptfBH8tkKzjpXqf1MICg3SFbZsJE63VQ5EKYJ0nH97MCZWBgZNNRjWw==, tarball: file:projects/util-internal-archive-client.tgz} name: '@rush-temp/util-internal-archive-client' version: 0.0.0 dependencies: @@ -5585,7 +5583,7 @@ packages: dev: false file:projects/util-internal-archive-layout.tgz: - resolution: {integrity: sha512-weUIUNCh9+vtDf2/S6YhwI+e4zSSBPWweIEZgHwjfmkueU6xMYMBjTQiNzxatenFZE86SW/yH3clF51cPia6Hw==, tarball: file:projects/util-internal-archive-layout.tgz} + resolution: {integrity: sha512-5cfaeuhdfbb7xKs3lEtD7NzJYpzjx0VERqTQV9s/H/WDa/dgUiH+Zv1HpvXUFcysc9g9TAERCtVxJp8XqBxOEA==, tarball: file:projects/util-internal-archive-layout.tgz} name: '@rush-temp/util-internal-archive-layout' version: 0.0.0 dependencies: diff --git a/substrate/substrate-dump/package.json b/substrate/substrate-dump/package.json index 552b566da..f3d5859e6 100644 --- a/substrate/substrate-dump/package.json +++ b/substrate/substrate-dump/package.json @@ -27,8 +27,10 @@ "@subsquid/util-internal-counters": "^1.3.0", "@subsquid/util-internal-fs": "^0.1.0", "@subsquid/util-internal-hex": "^1.2.0", + "@subsquid/util-internal-prometheus-server": "^1.2.0", "@subsquid/util-internal-range": "^0.0.0", - "commander": "^11.0.0" + "commander": "^11.0.0", + "prom-client": "14.2.0" }, "devDependencies": { "@types/node": "^18.16.17", diff --git a/substrate/substrate-dump/src/dumper.ts b/substrate/substrate-dump/src/dumper.ts index 6ce404041..072cb4978 100644 --- a/substrate/substrate-dump/src/dumper.ts +++ b/substrate/substrate-dump/src/dumper.ts @@ -14,6 +14,7 @@ import {printTimeInterval, Progress} from '@subsquid/util-internal-counters' import {createFs, Fs} from '@subsquid/util-internal-fs' import {assertRange, printRange, Range, rangeEnd} from '@subsquid/util-internal-range' import {MetadataWriter} from './metadata' +import { PrometheusServer } from './prometheus' export interface DumperOptions { @@ -26,6 +27,7 @@ export interface DumperOptions { lastBlock?: number withTrace?: boolean | string chunkSize: number + metricsPort?: number } @@ -82,6 +84,11 @@ export class Dumper { }) } + @def + prometheus() { + return new PrometheusServer(this.options.metricsPort ?? 3000); + } + ingest(range: Range): AsyncIterable { let request: DataRequest = { runtimeVersion: true, @@ -107,6 +114,7 @@ export class Dumper { let height = new Throttler(() => this.src().getFinalizedHeight(), 300_000) let chainHeight = await height.get() + this.prometheus().setLastBlock(this.options.lastBlock ? this.options.lastBlock : chainHeight); let progress = new Progress({ initialValue: this.range().from, @@ -193,11 +201,14 @@ export class Dumper { } } } else { - let archive = new ArchiveLayout(this.fs()) + const archive = new ArchiveLayout(this.fs()) + const prometheus = this.prometheus(); + if (this.options.metricsPort) await prometheus.serve(); await archive.appendRawBlocks({ blocks: (nextBlock, prevHash) => this.saveMetadata(this.process(nextBlock, prevHash)), range: this.range(), - chunkSize: this.options.chunkSize * 1024 * 1024 + chunkSize: this.options.chunkSize * 1024 * 1024, + onSuccessWrite: this.options.metricsPort ? ((block) => { prometheus.setLastSavedBlock(block); }) : undefined }) } } diff --git a/substrate/substrate-dump/src/main.ts b/substrate/substrate-dump/src/main.ts index 7478b3179..faa6a005a 100644 --- a/substrate/substrate-dump/src/main.ts +++ b/substrate/substrate-dump/src/main.ts @@ -22,6 +22,7 @@ runProgram(() => { program.option('--last-block ', 'Height of the last block to dump', nat) program.option('--with-trace [targets]', 'Fetch block trace') program.option('--chunk-size ', 'Data chunk size in megabytes', positiveInt, 32) + program.option('--metrics-port ', 'Port to serve metrics on', positiveInt) let args = program.parse().opts() as DumperOptions diff --git a/substrate/substrate-dump/src/prometheus.ts b/substrate/substrate-dump/src/prometheus.ts new file mode 100644 index 000000000..799b471c1 --- /dev/null +++ b/substrate/substrate-dump/src/prometheus.ts @@ -0,0 +1,46 @@ +import {createPrometheusServer, ListeningServer} from '@subsquid/util-internal-prometheus-server' +import promClient, { collectDefaultMetrics, Gauge, Registry } from 'prom-client'; + + +export class PrometheusServer { + private registry = new Registry() + private port?: number | string + private lastBlockGauge: Gauge; + private lastSavedBlockGauge: Gauge; + + constructor(port: number) { + this.port = port; + this.lastBlockGauge = new Gauge({ + name: 'sqd_last_block_total', + help: 'Last block available in the chain', + registers: [this.registry] + }); + + this.lastSavedBlockGauge = new Gauge({ + name: 'sqd_last_saved_block_total', + help: 'Last saved block', + registers: [this.registry] + }); + + collectDefaultMetrics({register: this.registry}) + } + + setLastBlock(block: number) { + this.lastBlockGauge.set(block); + } + + setLastSavedBlock(block: number) { + this.lastSavedBlockGauge.set(block); + } + + private getPort(): number | string { + return this.port == null + ? process.env.PROMETHEUS_PORT || 0 + : this.port + } + + + serve(): Promise { + return createPrometheusServer(this.registry, this.getPort()) + } +} diff --git a/util/util-internal-archive-layout/src/layout.ts b/util/util-internal-archive-layout/src/layout.ts index 9576f258f..d24f32c31 100644 --- a/util/util-internal-archive-layout/src/layout.ts +++ b/util/util-internal-archive-layout/src/layout.ts @@ -172,7 +172,8 @@ export class ArchiveLayout { args: { blocks: (nextBlock: number, prevHash?: string) => AsyncIterable range?: Range - chunkSize?: number + chunkSize?: number, + onSuccessWrite?: (lastBlock: number) => void } ): Promise { return this.append( @@ -190,6 +191,7 @@ export class ArchiveLayout { assertNotNull(lastBlock) ).transactDir('.', async fs => { let content = await out.end() + args.onSuccessWrite ? args.onSuccessWrite(lastBlock?.height || 0) : null; return fs.write('blocks.jsonl.gz', content) }) firstBlock = undefined From 5cf2abd4993fb0af70f3e4846e1b5a39bc0cceae Mon Sep 17 00:00:00 2001 From: Igor Mahov Date: Fri, 22 Sep 2023 20:15:05 +0300 Subject: [PATCH 2/8] fixes: fixed per review --- substrate/substrate-dump/package.json | 2 +- substrate/substrate-dump/src/dumper.ts | 7 +++++-- substrate/substrate-dump/src/main.ts | 2 +- substrate/substrate-dump/src/prometheus.ts | 8 +------- util/util-internal-archive-layout/src/layout.ts | 5 +++-- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/substrate/substrate-dump/package.json b/substrate/substrate-dump/package.json index f3d5859e6..3186fb549 100644 --- a/substrate/substrate-dump/package.json +++ b/substrate/substrate-dump/package.json @@ -30,7 +30,7 @@ "@subsquid/util-internal-prometheus-server": "^1.2.0", "@subsquid/util-internal-range": "^0.0.0", "commander": "^11.0.0", - "prom-client": "14.2.0" + "prom-client": "^14.2.0" }, "devDependencies": { "@types/node": "^18.16.17", diff --git a/substrate/substrate-dump/src/dumper.ts b/substrate/substrate-dump/src/dumper.ts index 072cb4978..d9afccd04 100644 --- a/substrate/substrate-dump/src/dumper.ts +++ b/substrate/substrate-dump/src/dumper.ts @@ -203,12 +203,15 @@ export class Dumper { } else { const archive = new ArchiveLayout(this.fs()) const prometheus = this.prometheus(); - if (this.options.metricsPort) await prometheus.serve(); + if (this.options.metricsPort) { + await prometheus.serve(); + this.log().info(`prometheus metrics are available on port {port}`) + } await archive.appendRawBlocks({ blocks: (nextBlock, prevHash) => this.saveMetadata(this.process(nextBlock, prevHash)), range: this.range(), chunkSize: this.options.chunkSize * 1024 * 1024, - onSuccessWrite: this.options.metricsPort ? ((block) => { prometheus.setLastSavedBlock(block); }) : undefined + onSuccessWrite: (block) => { prometheus.setLastSavedBlock(block); } }) } } diff --git a/substrate/substrate-dump/src/main.ts b/substrate/substrate-dump/src/main.ts index faa6a005a..0a883c084 100644 --- a/substrate/substrate-dump/src/main.ts +++ b/substrate/substrate-dump/src/main.ts @@ -22,7 +22,7 @@ runProgram(() => { program.option('--last-block ', 'Height of the last block to dump', nat) program.option('--with-trace [targets]', 'Fetch block trace') program.option('--chunk-size ', 'Data chunk size in megabytes', positiveInt, 32) - program.option('--metrics-port ', 'Port to serve metrics on', positiveInt) + program.option('--metrics-port ', 'Port to serve metrics on', nat) let args = program.parse().opts() as DumperOptions diff --git a/substrate/substrate-dump/src/prometheus.ts b/substrate/substrate-dump/src/prometheus.ts index 799b471c1..6e3e2e750 100644 --- a/substrate/substrate-dump/src/prometheus.ts +++ b/substrate/substrate-dump/src/prometheus.ts @@ -33,14 +33,8 @@ export class PrometheusServer { this.lastSavedBlockGauge.set(block); } - private getPort(): number | string { - return this.port == null - ? process.env.PROMETHEUS_PORT || 0 - : this.port - } - serve(): Promise { - return createPrometheusServer(this.registry, this.getPort()) + return createPrometheusServer(this.registry, this.port) } } diff --git a/util/util-internal-archive-layout/src/layout.ts b/util/util-internal-archive-layout/src/layout.ts index d24f32c31..f58c58dc4 100644 --- a/util/util-internal-archive-layout/src/layout.ts +++ b/util/util-internal-archive-layout/src/layout.ts @@ -191,8 +191,9 @@ export class ArchiveLayout { assertNotNull(lastBlock) ).transactDir('.', async fs => { let content = await out.end() - args.onSuccessWrite ? args.onSuccessWrite(lastBlock?.height || 0) : null; - return fs.write('blocks.jsonl.gz', content) + const writeResult = fs.write('blocks.jsonl.gz', content) + writeResult.then(() => args.onSuccessWrite ? args.onSuccessWrite(lastBlock?.height || 0) : null) + return writeResult }) firstBlock = undefined lastBlock = undefined From 44ce082481e82a4d01cc8269f103c8dda9c259aa Mon Sep 17 00:00:00 2001 From: Igor Mahov Date: Mon, 25 Sep 2023 13:58:13 +0300 Subject: [PATCH 3/8] fix: refactor per review --- .../feat-dump-metrics_2023-09-25-10-56.json | 10 ++++++++++ .../feat-dump-metrics_2023-09-25-10-56.json | 10 ++++++++++ substrate/substrate-dump/src/dumper.ts | 4 ++-- substrate/substrate-dump/src/prometheus.ts | 20 +++++++++---------- .../src/layout.ts | 15 ++++++++------ 5 files changed, 41 insertions(+), 18 deletions(-) create mode 100644 common/changes/@subsquid/substrate-dump/feat-dump-metrics_2023-09-25-10-56.json create mode 100644 common/changes/@subsquid/util-internal-archive-layout/feat-dump-metrics_2023-09-25-10-56.json diff --git a/common/changes/@subsquid/substrate-dump/feat-dump-metrics_2023-09-25-10-56.json b/common/changes/@subsquid/substrate-dump/feat-dump-metrics_2023-09-25-10-56.json new file mode 100644 index 000000000..64618cc51 --- /dev/null +++ b/common/changes/@subsquid/substrate-dump/feat-dump-metrics_2023-09-25-10-56.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/substrate-dump", + "comment": "feat: implement prometheus metrics", + "type": "minor" + } + ], + "packageName": "@subsquid/substrate-dump" +} \ No newline at end of file diff --git a/common/changes/@subsquid/util-internal-archive-layout/feat-dump-metrics_2023-09-25-10-56.json b/common/changes/@subsquid/util-internal-archive-layout/feat-dump-metrics_2023-09-25-10-56.json new file mode 100644 index 000000000..4c3efa125 --- /dev/null +++ b/common/changes/@subsquid/util-internal-archive-layout/feat-dump-metrics_2023-09-25-10-56.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@subsquid/util-internal-archive-layout", + "comment": "feat: implement onSuccessWrite callback", + "type": "minor" + } + ], + "packageName": "@subsquid/util-internal-archive-layout" +} \ No newline at end of file diff --git a/substrate/substrate-dump/src/dumper.ts b/substrate/substrate-dump/src/dumper.ts index d9afccd04..cbd5a7ea6 100644 --- a/substrate/substrate-dump/src/dumper.ts +++ b/substrate/substrate-dump/src/dumper.ts @@ -114,7 +114,7 @@ export class Dumper { let height = new Throttler(() => this.src().getFinalizedHeight(), 300_000) let chainHeight = await height.get() - this.prometheus().setLastBlock(this.options.lastBlock ? this.options.lastBlock : chainHeight); + this.prometheus().setChainHeight(chainHeight); let progress = new Progress({ initialValue: this.range().from, @@ -211,7 +211,7 @@ export class Dumper { blocks: (nextBlock, prevHash) => this.saveMetadata(this.process(nextBlock, prevHash)), range: this.range(), chunkSize: this.options.chunkSize * 1024 * 1024, - onSuccessWrite: (block) => { prometheus.setLastSavedBlock(block); } + onSuccessWrite: ({ blockRange: { to } }) => { prometheus.setLastWrittenBlock(to.height); } }) } } diff --git a/substrate/substrate-dump/src/prometheus.ts b/substrate/substrate-dump/src/prometheus.ts index 6e3e2e750..c2a9c94ae 100644 --- a/substrate/substrate-dump/src/prometheus.ts +++ b/substrate/substrate-dump/src/prometheus.ts @@ -5,19 +5,19 @@ import promClient, { collectDefaultMetrics, Gauge, Registry } from 'prom-client' export class PrometheusServer { private registry = new Registry() private port?: number | string - private lastBlockGauge: Gauge; - private lastSavedBlockGauge: Gauge; + private chainHeightGauge: Gauge; + private lastWrittenBlockGauge: Gauge; constructor(port: number) { this.port = port; - this.lastBlockGauge = new Gauge({ - name: 'sqd_last_block_total', + this.chainHeightGauge = new Gauge({ + name: 'sqd_dump_chain_height', help: 'Last block available in the chain', registers: [this.registry] }); - this.lastSavedBlockGauge = new Gauge({ - name: 'sqd_last_saved_block_total', + this.lastWrittenBlockGauge = new Gauge({ + name: 'sqd_dump_last_written_block', help: 'Last saved block', registers: [this.registry] }); @@ -25,12 +25,12 @@ export class PrometheusServer { collectDefaultMetrics({register: this.registry}) } - setLastBlock(block: number) { - this.lastBlockGauge.set(block); + setChainHeight(height: number) { + this.chainHeightGauge.set(height); } - setLastSavedBlock(block: number) { - this.lastSavedBlockGauge.set(block); + setLastWrittenBlock(block: number) { + this.lastWrittenBlockGauge.set(block); } diff --git a/util/util-internal-archive-layout/src/layout.ts b/util/util-internal-archive-layout/src/layout.ts index f58c58dc4..c25475ecf 100644 --- a/util/util-internal-archive-layout/src/layout.ts +++ b/util/util-internal-archive-layout/src/layout.ts @@ -173,7 +173,7 @@ export class ArchiveLayout { blocks: (nextBlock: number, prevHash?: string) => AsyncIterable range?: Range chunkSize?: number, - onSuccessWrite?: (lastBlock: number) => void + onSuccessWrite?: (args: { chunk: string, blockRange: { from: HashAndHeight, to: HashAndHeight } }) => void } ): Promise { return this.append( @@ -186,13 +186,16 @@ export class ArchiveLayout { let out = new GzipBuffer() async function save(): Promise { - await getNextChunk( - assertNotNull(firstBlock), - assertNotNull(lastBlock) - ).transactDir('.', async fs => { + const fBlock = assertNotNull(firstBlock); + const lBlock = assertNotNull(firstBlock); + const chunk = getNextChunk(fBlock, lBlock) + chunk.transactDir('.', async fs => { let content = await out.end() const writeResult = fs.write('blocks.jsonl.gz', content) - writeResult.then(() => args.onSuccessWrite ? args.onSuccessWrite(lastBlock?.height || 0) : null) + writeResult.then(() => args.onSuccessWrite ? args.onSuccessWrite({ + chunk: chunk.abs(), + blockRange: { from: fBlock, to: lBlock } + }) : null) return writeResult }) firstBlock = undefined From 785a35babc6f7adf004b31d10a398c3a54da3c1f Mon Sep 17 00:00:00 2001 From: Igor Mahov Date: Mon, 25 Sep 2023 14:00:20 +0300 Subject: [PATCH 4/8] fix: improve block range --- util/util-internal-archive-layout/src/layout.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/util/util-internal-archive-layout/src/layout.ts b/util/util-internal-archive-layout/src/layout.ts index c25475ecf..9ffdc217a 100644 --- a/util/util-internal-archive-layout/src/layout.ts +++ b/util/util-internal-archive-layout/src/layout.ts @@ -186,15 +186,17 @@ export class ArchiveLayout { let out = new GzipBuffer() async function save(): Promise { - const fBlock = assertNotNull(firstBlock); - const lBlock = assertNotNull(firstBlock); - const chunk = getNextChunk(fBlock, lBlock) + let blockRange = { + from: assertNotNull(firstBlock), + to: assertNotNull(lastBlock) + } + const chunk = getNextChunk(blockRange.from, blockRange.to) chunk.transactDir('.', async fs => { let content = await out.end() const writeResult = fs.write('blocks.jsonl.gz', content) writeResult.then(() => args.onSuccessWrite ? args.onSuccessWrite({ chunk: chunk.abs(), - blockRange: { from: fBlock, to: lBlock } + blockRange }) : null) return writeResult }) From af892c32269d033c2b8f8aed37829cdb942cc402 Mon Sep 17 00:00:00 2001 From: Igor Mahov Date: Tue, 26 Sep 2023 15:20:45 +0300 Subject: [PATCH 5/8] feat: add metrics for rpc requests --- substrate/substrate-dump/src/dumper.ts | 8 ++++++-- substrate/substrate-dump/src/prometheus.ts | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/substrate/substrate-dump/src/dumper.ts b/substrate/substrate-dump/src/dumper.ts index cbd5a7ea6..04eac4cf2 100644 --- a/substrate/substrate-dump/src/dumper.ts +++ b/substrate/substrate-dump/src/dumper.ts @@ -99,10 +99,11 @@ export class Dumper { : this.options.withTrace ? '' : undefined } - return this.src().getFinalizedBlocks([{ + const blocks = this.src().getFinalizedBlocks([{ range, request }]) + return blocks; } private async *process(from?: number, prevHash?: string): AsyncIterable { @@ -140,6 +141,9 @@ export class Dumper { ) } } + const metrics = this.rpc().getMetrics(); + this.prometheus().setSuccesfulRequestCount(metrics.requestsServed); + this.prometheus().setFailedRequestCount(metrics.connectionErrors); yield batch.blocks @@ -205,7 +209,7 @@ export class Dumper { const prometheus = this.prometheus(); if (this.options.metricsPort) { await prometheus.serve(); - this.log().info(`prometheus metrics are available on port {port}`) + this.log().info(`prometheus metrics are available on port ${this.options.metricsPort}`) } await archive.appendRawBlocks({ blocks: (nextBlock, prevHash) => this.saveMetadata(this.process(nextBlock, prevHash)), diff --git a/substrate/substrate-dump/src/prometheus.ts b/substrate/substrate-dump/src/prometheus.ts index c2a9c94ae..4316781c4 100644 --- a/substrate/substrate-dump/src/prometheus.ts +++ b/substrate/substrate-dump/src/prometheus.ts @@ -7,6 +7,7 @@ export class PrometheusServer { private port?: number | string private chainHeightGauge: Gauge; private lastWrittenBlockGauge: Gauge; + private rpcRequestsGauge: Gauge; constructor(port: number) { this.port = port; @@ -22,6 +23,13 @@ export class PrometheusServer { registers: [this.registry] }); + this.rpcRequestsGauge = new Gauge({ + name: 'sqd_rpc_requests_count', + help: 'Number of rpc requests of different kinds', + labelNames: ['kind'], + registers: [this.registry] + }); + collectDefaultMetrics({register: this.registry}) } @@ -33,6 +41,17 @@ export class PrometheusServer { this.lastWrittenBlockGauge.set(block); } + setSuccesfulRequestCount(requests: number) { + this.rpcRequestsGauge.set({ + 'kind': 'successful' + }, requests) + } + + setFailedRequestCount(requests: number) { + this.rpcRequestsGauge.set({ + 'kind': 'failed' + }, requests) + } serve(): Promise { return createPrometheusServer(this.registry, this.port) From 65f0caf7350ffa6bd0ca3525101355f642f24395 Mon Sep 17 00:00:00 2001 From: Igor Mahov Date: Wed, 27 Sep 2023 11:36:34 +0300 Subject: [PATCH 6/8] fixes per review --- substrate/substrate-dump/src/dumper.ts | 22 +++++++--------- substrate/substrate-dump/src/main.ts | 2 +- substrate/substrate-dump/src/prometheus.ts | 26 +++++++++---------- .../src/layout.ts | 13 +++++----- 4 files changed, 28 insertions(+), 35 deletions(-) diff --git a/substrate/substrate-dump/src/dumper.ts b/substrate/substrate-dump/src/dumper.ts index 04eac4cf2..5eac02d6e 100644 --- a/substrate/substrate-dump/src/dumper.ts +++ b/substrate/substrate-dump/src/dumper.ts @@ -27,7 +27,7 @@ export interface DumperOptions { lastBlock?: number withTrace?: boolean | string chunkSize: number - metricsPort?: number + metrics?: number } @@ -86,7 +86,7 @@ export class Dumper { @def prometheus() { - return new PrometheusServer(this.options.metricsPort ?? 3000); + return new PrometheusServer(this.options.metrics ?? 0, this.rpc()) } ingest(range: Range): AsyncIterable { @@ -99,11 +99,10 @@ export class Dumper { : this.options.withTrace ? '' : undefined } - const blocks = this.src().getFinalizedBlocks([{ + return this.src().getFinalizedBlocks([{ range, request }]) - return blocks; } private async *process(from?: number, prevHash?: string): AsyncIterable { @@ -115,7 +114,7 @@ export class Dumper { let height = new Throttler(() => this.src().getFinalizedHeight(), 300_000) let chainHeight = await height.get() - this.prometheus().setChainHeight(chainHeight); + this.prometheus().setChainHeight(chainHeight) let progress = new Progress({ initialValue: this.range().from, @@ -141,9 +140,6 @@ export class Dumper { ) } } - const metrics = this.rpc().getMetrics(); - this.prometheus().setSuccesfulRequestCount(metrics.requestsServed); - this.prometheus().setFailedRequestCount(metrics.connectionErrors); yield batch.blocks @@ -206,16 +202,16 @@ export class Dumper { } } else { const archive = new ArchiveLayout(this.fs()) - const prometheus = this.prometheus(); - if (this.options.metricsPort) { - await prometheus.serve(); - this.log().info(`prometheus metrics are available on port ${this.options.metricsPort}`) + const prometheus = this.prometheus() + if (this.options.metrics != null) { + const promServer = await prometheus.serve() + this.log().info(`prometheus metrics are available on port ${promServer.port}`) } await archive.appendRawBlocks({ blocks: (nextBlock, prevHash) => this.saveMetadata(this.process(nextBlock, prevHash)), range: this.range(), chunkSize: this.options.chunkSize * 1024 * 1024, - onSuccessWrite: ({ blockRange: { to } }) => { prometheus.setLastWrittenBlock(to.height); } + onSuccessWrite: ({ blockRange: { to } }) => { prometheus.setLastWrittenBlock(to.height) } }) } } diff --git a/substrate/substrate-dump/src/main.ts b/substrate/substrate-dump/src/main.ts index 0a883c084..b1af928e1 100644 --- a/substrate/substrate-dump/src/main.ts +++ b/substrate/substrate-dump/src/main.ts @@ -22,7 +22,7 @@ runProgram(() => { program.option('--last-block ', 'Height of the last block to dump', nat) program.option('--with-trace [targets]', 'Fetch block trace') program.option('--chunk-size ', 'Data chunk size in megabytes', positiveInt, 32) - program.option('--metrics-port ', 'Port to serve metrics on', nat) + program.option('--metrics ', 'Enable prometheus metrics server', nat) let args = program.parse().opts() as DumperOptions diff --git a/substrate/substrate-dump/src/prometheus.ts b/substrate/substrate-dump/src/prometheus.ts index 4316781c4..a969bfc42 100644 --- a/substrate/substrate-dump/src/prometheus.ts +++ b/substrate/substrate-dump/src/prometheus.ts @@ -1,3 +1,4 @@ +import { RpcClient } from '@subsquid/rpc-client'; import {createPrometheusServer, ListeningServer} from '@subsquid/util-internal-prometheus-server' import promClient, { collectDefaultMetrics, Gauge, Registry } from 'prom-client'; @@ -9,7 +10,7 @@ export class PrometheusServer { private lastWrittenBlockGauge: Gauge; private rpcRequestsGauge: Gauge; - constructor(port: number) { + constructor(port: number, rpc: RpcClient) { this.port = port; this.chainHeightGauge = new Gauge({ name: 'sqd_dump_chain_height', @@ -27,7 +28,16 @@ export class PrometheusServer { name: 'sqd_rpc_requests_count', help: 'Number of rpc requests of different kinds', labelNames: ['kind'], - registers: [this.registry] + registers: [this.registry], + collect() { + const metrics = rpc.getMetrics(); + this.set({ + kind: 'successful' + }, metrics.requestsServed); + this.set({ + kind: 'failed' + }, metrics.connectionErrors); + } }); collectDefaultMetrics({register: this.registry}) @@ -41,18 +51,6 @@ export class PrometheusServer { this.lastWrittenBlockGauge.set(block); } - setSuccesfulRequestCount(requests: number) { - this.rpcRequestsGauge.set({ - 'kind': 'successful' - }, requests) - } - - setFailedRequestCount(requests: number) { - this.rpcRequestsGauge.set({ - 'kind': 'failed' - }, requests) - } - serve(): Promise { return createPrometheusServer(this.registry, this.port) } diff --git a/util/util-internal-archive-layout/src/layout.ts b/util/util-internal-archive-layout/src/layout.ts index 9ffdc217a..bf4285c1f 100644 --- a/util/util-internal-archive-layout/src/layout.ts +++ b/util/util-internal-archive-layout/src/layout.ts @@ -193,13 +193,12 @@ export class ArchiveLayout { const chunk = getNextChunk(blockRange.from, blockRange.to) chunk.transactDir('.', async fs => { let content = await out.end() - const writeResult = fs.write('blocks.jsonl.gz', content) - writeResult.then(() => args.onSuccessWrite ? args.onSuccessWrite({ - chunk: chunk.abs(), - blockRange - }) : null) - return writeResult - }) + return fs.write('blocks.jsonl.gz', content) + }).then(() => args.onSuccessWrite?.({ + chunk: chunk.abs(), + blockRange + })) + firstBlock = undefined lastBlock = undefined out = new GzipBuffer() From 74a6bfb435634b122d7bdd77ed672690203aac3e Mon Sep 17 00:00:00 2001 From: Igor Mahov Date: Wed, 27 Sep 2023 18:48:05 +0300 Subject: [PATCH 7/8] Improvements per review --- substrate/substrate-dump/src/prometheus.ts | 10 ++++++---- util/util-internal-archive-layout/src/layout.ts | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/substrate/substrate-dump/src/prometheus.ts b/substrate/substrate-dump/src/prometheus.ts index a969bfc42..000f0b310 100644 --- a/substrate/substrate-dump/src/prometheus.ts +++ b/substrate/substrate-dump/src/prometheus.ts @@ -25,17 +25,19 @@ export class PrometheusServer { }); this.rpcRequestsGauge = new Gauge({ - name: 'sqd_rpc_requests_count', + name: 'sqd_dump_rpc_requests_count', help: 'Number of rpc requests of different kinds', - labelNames: ['kind'], + labelNames: ['kind', 'url'], registers: [this.registry], collect() { const metrics = rpc.getMetrics(); this.set({ - kind: 'successful' + kind: 'successful', + url: rpc.url }, metrics.requestsServed); this.set({ - kind: 'failed' + kind: 'failed', + url: rpc.url }, metrics.connectionErrors); } }); diff --git a/util/util-internal-archive-layout/src/layout.ts b/util/util-internal-archive-layout/src/layout.ts index bf4285c1f..8d4ce7379 100644 --- a/util/util-internal-archive-layout/src/layout.ts +++ b/util/util-internal-archive-layout/src/layout.ts @@ -191,7 +191,7 @@ export class ArchiveLayout { to: assertNotNull(lastBlock) } const chunk = getNextChunk(blockRange.from, blockRange.to) - chunk.transactDir('.', async fs => { + await chunk.transactDir('.', async fs => { let content = await out.end() return fs.write('blocks.jsonl.gz', content) }).then(() => args.onSuccessWrite?.({ From 41fc80f07f63205bed715112ac4e1765cdbf1692 Mon Sep 17 00:00:00 2001 From: Igor Mahov Date: Wed, 27 Sep 2023 19:08:36 +0300 Subject: [PATCH 8/8] fix: change kind label --- substrate/substrate-dump/src/prometheus.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substrate/substrate-dump/src/prometheus.ts b/substrate/substrate-dump/src/prometheus.ts index 000f0b310..ab60a1233 100644 --- a/substrate/substrate-dump/src/prometheus.ts +++ b/substrate/substrate-dump/src/prometheus.ts @@ -32,11 +32,11 @@ export class PrometheusServer { collect() { const metrics = rpc.getMetrics(); this.set({ - kind: 'successful', + kind: 'success', url: rpc.url }, metrics.requestsServed); this.set({ - kind: 'failed', + kind: 'failure', url: rpc.url }, metrics.connectionErrors); }