feat: add env vars logging, rpc startup checks, new metrics

* feat: add ENV logs

* feat: push open ENV vars to prom

* refactor: simplify Metrics class

* refactor: use the openKeys in the startup metrics

* feat: add startup check RPC

* fix: log level

* feat: check all rpc

* refactor: dirs

* feat: remove RUN_STARTUP_CHECKS from 'yarn dev'

* feat: pass startup-rpc-checks from mjs to Metrics class (ts source code)

* feat: rpc startup checks metrics

* feat: viem instead of ethers

* chore: remove debug

* refactor: rpc timeout 10 seconds

* feat: remove stopping the app

consts/metrics.ts

@@ -2,6 +2,7 @@ export const METRICS_PREFIX = 'eth_stake_widget_ui_';
 
 export const enum METRIC_NAMES {
   REQUESTS_TOTAL = 'requests_total',
+  STARTUP_CHECKS_RPC = 'startup_checks_rpc',
   API_RESPONSE = 'api_response',
   ETH_CALL_ADDRESS_TO = 'eth_call_address_to',
   SSR_COUNT = 'ssr_count',

next.config.mjs

@@ -1,9 +1,17 @@
 import NextBundleAnalyzer from '@next/bundle-analyzer';
 import buildDynamics from './scripts/build-dynamics.mjs';
+import { logEnvironmentVariables } from './scripts/log-environment-variables.mjs';
 import generateBuildId from './scripts/generate-build-id.mjs';
+import { startupCheckRPCs } from './scripts/startup-checks/rpc.mjs';
 
+logEnvironmentVariables();
 buildDynamics();
 
+if (process.env.RUN_STARTUP_CHECKS === 'true' && typeof window === 'undefined') {
+  void startupCheckRPCs();
+}
+
+
 // https://nextjs.org/docs/pages/api-reference/next-config-js/basePath
 const basePath = process.env.BASE_PATH;
 

package.json

@@ -10,7 +10,7 @@
     "build": "NODE_OPTIONS='--no-warnings=ExperimentalWarning' next build",
     "build:analyze": "ANALYZE_BUNDLE=true yarn build",
     "build:ipfs": "IPFS_MODE=true yarn build && IPFS_MODE=true next export",
-    "start": "NODE_ENV=production node -r next-logger --no-warnings=ExperimentalWarning server.mjs",
+    "start": "NODE_ENV=production RUN_STARTUP_CHECKS=true node -r next-logger --no-warnings=ExperimentalWarning server.mjs",
     "lint": "eslint --ext ts,tsx,js,mjs .",
     "lint:fix": "yarn lint --fix",
     "types": "tsc --noEmit",

pages/api/metrics.ts

@@ -1,13 +1,13 @@
 import { wrapRequest as wrapNextRequest } from '@lidofinance/next-api-wrapper';
+import { metricsFactory } from '@lidofinance/next-pages';
+
 import { API_ROUTES } from 'consts/api';
 import {
   responseTimeMetric,
   errorAndCacheDefaultWrappers,
   rateLimit,
 } from 'utilsApi';
-
 import Metrics from 'utilsApi/metrics';
-import { metricsFactory } from '@lidofinance/next-pages';
 
 const metrics = metricsFactory({
   registry: Metrics.registry,

scripts/log-environment-variables.mjs

@@ -0,0 +1,80 @@
+export const openKeys = [
+  'SELF_ORIGIN',
+  'ROOT_ORIGIN',
+  'DOCS_ORIGIN',
+  'HELP_ORIGIN',
+  'RESEARCH_ORIGIN',
+
+  'SUPPORTED_CHAINS',
+  'DEFAULT_CHAIN',
+
+  'CSP_TRUSTED_HOSTS',
+  'CSP_REPORT_ONLY',
+  'CSP_REPORT_URI',
+
+  'ENABLE_QA_HELPERS',
+
+  'REWARDS_BACKEND',
+
+  'RATE_LIMIT',
+  'RATE_LIMIT_TIME_FRAME',
+
+  'ETH_API_BASE_PATH',
+  'WQ_API_BASE_PATH',
+  'MATOMO_URL',
+  'WALLETCONNECT_PROJECT_ID',
+  'REWARDS_BACKEND_BASE_PATH',
+];
+
+export const secretKeys = [
+  'EL_RPC_URLS_1',
+  'EL_RPC_URLS_5',
+  'EL_RPC_URLS_17000',
+  'EL_RPC_URLS_11155111',
+]
+
+
+export const logOpenEnvironmentVariables = () => {
+  console.log('---------------------------------------------');
+  console.log('Log environment variables (without secrets):');
+  console.log('---------------------------------------------');
+
+  for (const key of openKeys) {
+    if (!process.env.hasOwnProperty(key)) {
+      console.error(`${key} - ERROR (not exist in process.env)`);
+      continue;
+    }
+
+    console.info(`${key} = ${process.env[key]}`);
+  }
+
+  console.log('---------------------------------------------');
+  console.log('');
+};
+
+export const logSecretEnvironmentVariables = () => {
+  console.log('---------------------------------------------');
+  console.log('Log secret environment variables:');
+  console.log('---------------------------------------------');
+
+  // console.log('process.env:', process.env)
+  for (const key of secretKeys) {
+    if (!process.env.hasOwnProperty(key)) {
+      console.error(`Secret ${key} - ERROR (not exist in process.env)`);
+      continue;
+    }
+
+    if (process.env[key].length > 0) {
+      console.info(`Secret ${key} - OK (exist and not empty)`);
+    } else {
+      console.warn(`Secret ${key} - WARN (exist but empty)`);
+    }
+  }
+
+  console.log('---------------------------------------------');
+};
+
+export const logEnvironmentVariables = () => {
+  logOpenEnvironmentVariables();
+  logSecretEnvironmentVariables();
+};

scripts/startup-checks/rpc.mjs

@@ -0,0 +1,78 @@
+import { createClient, http } from 'viem';
+import { getChainId } from 'viem/actions'
+
+// Safely initialize a global variable
+let globalRPCCheckResults = globalThis.__rpcCheckResults || [];
+globalThis.__rpcCheckResults = globalRPCCheckResults;
+
+export const BROKEN_URL = 'BROKEN_URL';
+export const RPC_TIMEOUT_SECONDS = 10_000;
+
+const pushRPCCheckResult = (domain, success) => {
+  globalRPCCheckResults.push({ domain, success });
+};
+
+export const getRPCCheckResults = () => globalThis.__rpcCheckResults || [];
+
+const getRpcUrls = (chainId) => {
+  const rpcUrls = process.env[`EL_RPC_URLS_${chainId}`]?.split(',');
+  return rpcUrls?.filter((url) => url);
+};
+
+export const startupCheckRPCs = async () => {
+  console.info('[startupCheckRPCs] Starting...');
+
+  try {
+    const defaultChain = parseInt(process.env.DEFAULT_CHAIN, 10);
+    const rpcUrls = getRpcUrls(defaultChain);
+
+    if (!rpcUrls || rpcUrls.length === 0) {
+      throw new Error('[startupCheckRPCs] No RPC URLs found!');
+    }
+
+    let errorCount = 0;
+
+    for (const url of rpcUrls) {
+      let domain;
+      try {
+        domain = new URL(url).hostname;
+      } catch (err) {
+        errorCount += 1;
+        console.error('There is a broken URL.');
+        pushRPCCheckResult(BROKEN_URL, false);
+        continue;
+      }
+
+      try {
+        const client = createClient({
+          transport: http(url, { retryCount: 0, timeout: RPC_TIMEOUT_SECONDS })
+        });
+
+        const chainId = await getChainId(client);
+
+        if (defaultChain === chainId) {
+          pushRPCCheckResult(domain, true);
+          console.info(`[startupCheckRPCs] RPC ${domain} works!`);
+        } else {
+          throw(`[startupCheckRPCs] RPC ${domain} does not work!`);
+        }
+      } catch (err) {
+        errorCount += 1;
+        pushRPCCheckResult(domain, false);
+        console.error(`[startupCheckRPCs] Error with RPC ${domain}:`);
+        console.error(String(err).replaceAll(rpcUrls, domain));
+        console.error(`[startupCheckRPCs] Timeout: ${RPC_TIMEOUT_SECONDS} seconds`);
+      }
+    }
+
+    if (errorCount > 0) {
+      console.info(`[startupCheckRPCs] Number of working RPCs: ${rpcUrls.length - errorCount}`);
+      console.info(`[startupCheckRPCs] Number of broken RPCs: ${errorCount}`);
+    } else {
+      console.info('[startupCheckRPCs] All RPC works!');
+    }
+  } catch (err) {
+    console.error('[startupCheckRPCs] Error during startup check:');
+    console.error(err);
+  }
+};

utilsApi/metrics/metrics.ts

@@ -1,10 +1,7 @@
 import { collectDefaultMetrics, Registry } from 'prom-client';
-import { collectStartupMetrics } from '@lidofinance/api-metrics';
-
-import { config } from 'config';
 import { METRICS_PREFIX } from 'consts/metrics';
-import buildInfoJson from 'build-info.json';
 
+import { collectStartupMetrics } from './startup-metrics';
 import { RequestMetrics } from './request';
 
 class Metrics {
@@ -14,23 +11,9 @@ class Metrics {
   request = new RequestMetrics(this.registry);
 
   constructor() {
-    this.collectStartupMetricsInit();
+    collectStartupMetrics(this.registry);
     collectDefaultMetrics({ prefix: METRICS_PREFIX, register: this.registry });
   }
-
-  collectStartupMetricsInit() {
-    collectStartupMetrics({
-      prefix: METRICS_PREFIX,
-      registry: this.registry,
-      defaultChain: `${config.defaultChain}`,
-      supportedChains: config.supportedChains.map(
-        (chain: number) => `${chain}`,
-      ),
-      version: buildInfoJson.version,
-      commit: buildInfoJson.commit,
-      branch: buildInfoJson.branch,
-    });
-  }
 }
 
 export default new Metrics();

utilsApi/metrics/startup-checks.ts

@@ -0,0 +1,22 @@
+import { Counter, Registry } from 'prom-client';
+import { METRICS_PREFIX, METRIC_NAMES } from 'consts/metrics';
+
+export class StartupChecksRPCMetrics {
+  requestCounter: Counter<'rpc_domain' | 'success'>;
+
+  constructor(public registry: Registry) {
+    this.requestCounter = this.requestsCounterInit();
+  }
+
+  requestsCounterInit() {
+    const requestsCounterName =
+      METRICS_PREFIX + METRIC_NAMES.STARTUP_CHECKS_RPC;
+
+    return new Counter({
+      name: requestsCounterName,
+      help: 'The total number of RPC checks after the app started.',
+      labelNames: ['rpc_domain', 'success'],
+      registers: [this.registry],
+    });
+  }
+}

utilsApi/metrics/startup-metrics.ts

@@ -0,0 +1,54 @@
+import { Gauge, type Registry } from 'prom-client';
+import { collectStartupMetrics as collectBuildInfoMetrics } from '@lidofinance/api-metrics';
+
+import buildInfoJson from 'build-info.json';
+import { openKeys } from 'scripts/log-environment-variables.mjs';
+import { getRPCCheckResults } from 'scripts/startup-checks/rpc.mjs';
+
+import { config } from 'config';
+import { METRICS_PREFIX } from 'consts/metrics';
+
+import { StartupChecksRPCMetrics } from './startup-checks';
+
+const collectStartupChecksRPCMetrics = (registry: Registry): void => {
+  const rpcMetrics = new StartupChecksRPCMetrics(registry);
+
+  getRPCCheckResults().forEach(
+    (_check: { domain: string; success: boolean }) => {
+      rpcMetrics.requestCounter
+        .labels(_check.domain, _check.success.toString())
+        .inc();
+    },
+  );
+};
+
+const collectEnvInfoMetrics = (registry: Registry): void => {
+  const labelPairs = openKeys.map((key) => ({
+    name: key,
+    value: process.env[key] ?? '',
+  }));
+
+  const envInfo = new Gauge({
+    name: METRICS_PREFIX + 'env_info',
+    help: 'Environment variables of the current runtime',
+    labelNames: labelPairs.map((pair) => pair.name),
+    registers: [registry],
+  });
+  envInfo.labels(...labelPairs.map((pair) => pair.value)).set(1);
+};
+
+export const collectStartupMetrics = (registry: Registry): void => {
+  collectEnvInfoMetrics(registry);
+
+  collectBuildInfoMetrics({
+    prefix: METRICS_PREFIX,
+    registry: registry,
+    defaultChain: `${config.defaultChain}`,
+    supportedChains: config.supportedChains.map((chain: number) => `${chain}`),
+    version: buildInfoJson.version,
+    commit: buildInfoJson.commit,
+    branch: buildInfoJson.branch,
+  });
+
+  collectStartupChecksRPCMetrics(registry);
+};