feat: add prometheus metrics endpoint in dedicated server & port (#…

…1232)

* feat: add prometheus `metrics` endpoint in dedicated server & port
- add separate express instance and port for prometheus metrics
- expose the default prometheus metrics
- add a custom metric of type `counter` that increments in every http error

* fix lint errors by using an IIFE & adding void operator
- remove the `return` keyword
- add an IIFE inside the callback
- add the `void` operator

* update dep prom-client

* add prometheus flags & fix for process.kill
- add a prometheus flag that runs the prometheus server & enables the metrics endpoint
- add a prometheus-port flag so that the user can choose a custom port to run the metrics endpoint
- change the code in killAll so that there is no `Object is possibly undefined` error

* code refactor, help page & README
- add prometheus options and description under help flag
- add `metricsApp` and increased `httpErrorCounter` only if prometheus flag is true
- move logger and register inside functions
- docs: add a section for prometheus in the README

* remove jest unnecessary flags

---------

Co-authored-by: tarikgul <[email protected]>

README.md

@@ -242,6 +242,26 @@ file you can `symlink` it with `.env.test`. For example you could run
 `ln -s .env.myEnv .env.test && yarn start:log-rpc` to use `.env.myEnv` to set ENV variables. (see linux
 commands `ln` and `unlink` for more info.)
 
+### Prometheus server
+Prometheus metrics can be enabled by running sidecar with the following flag :
+
+```bash
+yarn start --prometheus
+```
+
+You can also define a custom port by running :
+
+```bash
+yarn start --prometheus --prometheus-port=<YOUR_CUSTOM_PORT>
+```
+
+The metrics endpoint can then be accessed :
+- on the default port : `http://127.0.0.1:9100/metrics` or
+- on your custom port if you defined one : `http://127.0.0.1:<YOUR_CUSTOM_PORT>/metrics`
+
+That way you will have access to the default prometheus metrics and one extra custom metric called `sas_http_errors` (of type counter). This counter is increased by 1 every time an http error has occured in sidecar.
+
+
 ## Debugging fee and staking payout calculations
 
 It is possible to get more information about the fee and staking payout calculation process logged to
@@ -258,7 +278,7 @@ CALC_DEBUG=1 sh calc/build.sh
 
 ## Chain integration guide
 
-[Click here for chain integration guide.](./guides/CHAIN_INTEGRATION.md))
+[Click here for chain integration guide.](./guides/CHAIN_INTEGRATION.md)
 
 ## Docker
 

package.json

@@ -42,9 +42,9 @@
     "start:latest-e2e-scripts": "yarn build:scripts && node scripts/build/runLatestE2eTests.js",
     "start:historical-e2e-tests": "yarn build:e2e-tests && node ./e2e-tests/build/historical/historical.js --config=./e2e-tests/jest.config.js",
     "start:historical-e2e-scripts": "yarn build:scripts && node scripts/build/runHistoricalE2eTests.js",
-    "test": "NODE_ENV=test substrate-exec-jest --detectOpenHandles",
+    "test": "NODE_ENV=test substrate-exec-jest",
     "test:watch": "NODE_ENV=test substrate-exec-jest --watch",
-    "test:ci": "NODE_ENV=test substrate-exec-jest --runInBand",
+    "test:ci": "NODE_ENV=test substrate-exec-jest",
     "test:latest-e2e-tests": "yarn start:latest-e2e-scripts",
     "test:historical-e2e-tests": "yarn start:historical-e2e-scripts",
     "test:test-release": "yarn build:scripts && node scripts/build/runYarnPack.js"
@@ -60,6 +60,7 @@
     "express-winston": "^4.2.0",
     "http-errors": "^2.0.0",
     "lru-cache": "^7.13.1",
+    "prom-client": "^14.2.0",
     "rxjs": "^7.5.6",
     "winston": "^3.8.1"
   },

scripts/sidecarScriptApi.ts

@@ -56,9 +56,12 @@ export const killAll = (procs: ProcsType): void => {
 		if (!procs[key].killed) {
 			try {
 				console.log(`Killing ${key}`);
+				const ppid = procs[key].pid;
 				// Kill child and all its descendants.
-				process.kill(-procs[key].pid, 'SIGTERM');
-				process.kill(-procs[key].pid, 'SIGKILL');
+				if (ppid != undefined) {
+					process.kill(-ppid, 'SIGTERM');
+					process.kill(-ppid, 'SIGKILL');
+				}
 			} catch (e) {
 				/**
 				 * The error we are catching here silently, is when `-procs[key].pid` takes

src/main.ts

@@ -32,6 +32,7 @@ import { Log } from './logging/Log';
 import * as middleware from './middleware';
 import { parseArgs } from './parseArgs';
 import { SidecarConfig } from './SidecarConfig';
+import Metrics_App from './util/metrics';
 
 async function main() {
 	const { config } = SidecarConfig;
@@ -93,6 +94,16 @@ async function main() {
 
 	server.keepAliveTimeout = config.EXPRESS.KEEP_ALIVE_TIMEOUT;
 	server.headersTimeout = config.EXPRESS.KEEP_ALIVE_TIMEOUT + 5000;
+
+	if (args.prometheus) {
+		// Create Metrics App
+		const metricsApp = new Metrics_App({
+			port: 9100,
+			host: config.EXPRESS.HOST,
+		});
+		// Start the Metrics server
+		metricsApp.listen();
+	}
 }
 
 /**

src/middleware/error/httpErrorMiddleware.ts

@@ -18,6 +18,8 @@ import { ErrorRequestHandler } from 'express';
 import { HttpError } from 'http-errors';
 
 import { Log } from '../../logging/Log';
+import { parseArgs } from '../../parseArgs';
+import { httpErrorCounter } from '../../util/metrics';
 /**
  * Handle HttpError instances.
  *
@@ -38,15 +40,17 @@ export const httpErrorMiddleware: ErrorRequestHandler = (
 	if (res.headersSent || !(err instanceof HttpError)) {
 		return next(err);
 	}
-
+	const args = parseArgs();
 	const code = err.status;
 
 	const info = {
 		code,
 		message: err.message,
 		stack: err.stack,
 	};
-
+	if (args.prometheus) {
+		httpErrorCounter.inc();
+	}
 	Log.logger.error(info);
 
 	res.status(code).send(info);

src/middleware/validate/validateBooleanMiddleware.spec.ts

@@ -20,7 +20,7 @@ import { BadRequest } from 'http-errors';
 import { doesNotErrorWith, errorsWith } from './util';
 import { validateBooleanMiddleware } from './validateBooleanMiddleware';
 
-describe('validaeBooleanMiddleware', () => {
+describe('validateBooleanMiddleware', () => {
 	doesNotErrorWith(
 		'no query params in path',
 		{

src/parseArgs.ts

@@ -19,7 +19,19 @@ import { ArgumentParser, Namespace } from 'argparse';
 export const parseArgs = (): Namespace => {
 	const parser = new ArgumentParser();
 
-	parser.add_argument('-v', '--version', { action: 'store_true' });
+	parser.add_argument('-v', '--version', {
+		action: 'store_true',
+		help: 'print substrate-api-sidecar version',
+	});
+	parser.add_argument('-p', '--prometheus', {
+		action: 'store_true',
+		help: 'enable the prometheus metrics endpoint',
+	});
+	parser.add_argument('-pp', '--prometheus-port', {
+		type: 'int',
+		default: 9100,
+		help: 'specify the port number on which the prometheus metrics are exposed [default: 9100]',
+	});
 
 	return parser.parse_args() as Namespace;
 };

src/util/metrics.ts

@@ -0,0 +1,60 @@
+import express from 'express';
+import { Application, Request, Response } from 'express';
+import client from 'prom-client';
+
+import { Log } from '../logging/Log';
+import { parseArgs } from '../parseArgs';
+
+export const httpErrorCounter = new client.Counter({
+	name: 'sas_http_errors',
+	help: 'Number of HTTP Errors',
+});
+
+interface IAppConfiguration {
+	port: number;
+	host: string;
+}
+
+export default class Metrics_App {
+	private app: Application;
+	private readonly port: number;
+	private readonly host: string;
+
+	/**
+	 * @param appConfig configuration for app.
+	 */
+	constructor({ host }: IAppConfiguration) {
+		const args = parseArgs();
+
+		this.port = Number(args.prometheus_port);
+		this.app = express();
+		this.host = host;
+
+		this.metricsEndpoint();
+	}
+
+	listen(): void {
+		const { logger } = Log;
+		this.app.listen(this.port, this.host, () => {
+			logger.info(
+				`Metrics Server started at http://${this.host}:${this.port}/`
+			);
+		});
+	}
+
+	/**
+	 * Mount the metrics endpoint.
+	 */
+	private metricsEndpoint() {
+		const register = new client.Registry();
+		register.registerMetric(httpErrorCounter);
+		client.collectDefaultMetrics({ register, prefix: 'sas_' });
+		// Set up the metrics endpoint
+		this.app.get('/metrics', (_req: Request, res: Response) => {
+			void (async () => {
+				res.set('Content-Type', register.contentType);
+				res.send(await register.metrics());
+			})();
+		});
+	}
+}