merge main

k8s/production/deployment.yaml

@@ -35,6 +35,11 @@ spec:
                 secretKeyRef:
                   name: observatory-service-prod-secrets
                   key: SLACK_CHANNEL_URL
+            - name: SLACK_CHANNEL_URL_P0
+              valueFrom:
+                secretKeyRef:
+                  name: observatory-service-prod-secrets
+                  key: SLACK_CHANNEL_URL_P0
             - name: GRAFANA_API_KEY
               valueFrom:
                 secretKeyRef:

prisma/migrations/20240408042925_remove_chain_name_column/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `chain_name` on the `response_codes` table. All the data in the column will be lost.
+
+*/
+-- DropIndex
+DROP INDEX "response_codes_chain_name_type_idx";
+
+-- AlterTable
+ALTER TABLE "response_codes" DROP COLUMN "chain_name";

prisma/schema.prisma

@@ -8,25 +8,23 @@ datasource db {
 }
 
 model ResponseCode {
-  id               Int      @id @default(autoincrement())
-  createdAt        DateTime @default(now()) @db.Timestamptz(3)
-  updatedAt        DateTime @updatedAt @db.Timestamptz(3)
-  chainName        String?  @map("chain_name")
+  id               Int          @id @default(autoincrement())
+  createdAt        DateTime     @default(now()) @db.Timestamptz(3)
+  updatedAt        DateTime     @updatedAt @db.Timestamptz(3)
   type             Types
   httpResponseCode Int
   url              String
   responseTime     Int?
-  chainId          String?  @map("chain_id")
-  priority         Int?     @default(0)
-  endpointType      EndpointType @default(latest_block)
+  chainId          String?      @map("chain_id")
+  priority         Int?         @default(0)
+  endpointType     EndpointType @default(latest_block)
   provider         String?
   errorMessage     String?
-  chainPriority    Int    @default(1)
+  chainPriority    Int          @default(1)
 
-  @@index([chainName, type])
+  @@index([type])
   @@index([createdAt])
   @@index([chainId])
-  @@index([type])
   @@index([httpResponseCode])
   @@index([responseTime])
   @@index([url])

src/app.ts

@@ -13,6 +13,7 @@ import './cron/data-accuracy-check';
 import './cron/nms-p1-last-hour-downtime-check';
 import './cron/numia-last-hour-downtime-check';
 import './cron/allnodes-last-hour-downtime-check';
+import './cron/nms-api-check';
 import { Pinger } from './pinger';
 import sleep from './sleep';
 import prometheus from 'prom-client';
@@ -61,8 +62,8 @@ async function startIndividualNodePinger(): Promise<void> {
         logger.informational('Starting a new iteration of Individual Node Pinger.');
         const pinger = Container.get(Pinger.token);
 
-        const handlePing = async (url: string, chainId: string, chainName: string | undefined) => {
-          return pinger.ping(url, chainName, Types.SINGULAR, chainId).catch((error) => {
+        const handlePing = async (url: string, chainId: string) => {
+          return pinger.ping(url, Types.SINGULAR, chainId).catch((error) => {
             logger.error(`Error pinging ${url} for chainId ${chainId}: ${error.message}`);
             return null; // Return a value to keep the array's structure consistent
           });
@@ -72,11 +73,10 @@ async function startIndividualNodePinger(): Promise<void> {
 
         for (const nodeData of jsonData) {
           const chainId = nodeData.chainId;
-          const chainName = nodeData.chainName;
           const nodes = nodeData.nodeList;
 
           for (const url of nodes) {
-            promisesArr.push(handlePing(url, chainId, chainName));
+            promisesArr.push(handlePing(url, chainId));
           }
         }
 
@@ -126,21 +126,17 @@ async function startSingularPaidNodePinger(): Promise<void> {
     return;
   }
 
-  const chainIdToNameMap = new Map(
-    jsonData.map((node: { chainId: string; chainName: string }) => [node.chainId, node.chainName]),
-  );
 
   while (true) {
     try {
       const startTime = Date.now();
       logger.informational('Starting a new iteration of Singular Paid Node Pinger.');
       const pinger = Container.get(Pinger.token); // Assuming Pinger is set up in Container
 
-      const handlePing = async (url: string, chainId: string, chainName: string, provider: string | null) => {
+      const handlePing = async (url: string, chainId: string,  provider: string | null) => {
         return pinger
           .ping(
             url,
-            chainName,
             Types.SINGULAR_PAID,
             chainId,
             '/cosmos/base/tendermint/v1beta1/blocks/latest',
@@ -163,14 +159,13 @@ async function startSingularPaidNodePinger(): Promise<void> {
           // Here's the critical check
           if (!Array.isArray(nodes)) {
             logger.error(
-              `Node list for provider "${provider}" in chain "${nodeData.chainName}" (${chainId}) is not an array.`,
+              `Node list for provider "${provider}" in chain  (${chainId}) is not an array.`,
             );
             continue; // Skip this iteration
           }
 
-          const chainName = chainIdToNameMap.get(chainId);
           for (const url of nodes) {
-            promisesArr.push(handlePing(url, chainId, chainName!, provider));
+            promisesArr.push(handlePing(url, chainId, provider));
           }
         }
       }
@@ -239,7 +234,7 @@ async function startEcostakePinger(): Promise<void> {
         const details = chainDetails as any;
         const url = `https://rest.cosmos.directory/${details.chainRegistryPath}`;
         arr.push(
-          pinger.ping(url, details.chainName, Types.ECOSTAKE, details.chainId).catch((error) => {
+          pinger.ping(url,  Types.ECOSTAKE, details.chainId).catch((error) => {
             logger.error(`Error pinging ${url} for chainId ${details.chainId}: ${error.message}`);
             return null;
           }),
@@ -364,10 +359,6 @@ async function startNMSPinger(nmsRunType: Types): Promise<void> {
     return;
   }
 
-  const chainIdToNameMap = new Map(
-    jsonData.map((node: { chainId: string; chainName: string }) => [node.chainId, node.chainName]),
-  );
-
   if (EnvVars.getNodeEnv() === 'test') return;
   let promisesArr: Promise<Prisma.ResponseCodeCreateInput>[] = [];
   while (true) {
@@ -381,11 +372,10 @@ async function startNMSPinger(nmsRunType: Types): Promise<void> {
       for (let i = 0; i < chainIds.length; i++) {
         const chainId = chainIds[i] || '';
         const nodes = jsonData[chainId!];
-        const chainName = chainIdToNameMap.get(chainId);
         const urls = await nmsGetNodeURL(nodes, nmsRunType);
         urls.forEach(({ url, priority }) => {
           promisesArr.push(
-            pinger.ping(url, chainName, nmsRunType, chainId, '/cosmos/base/tendermint/v1beta1/blocks/latest', priority),
+            pinger.ping(url, nmsRunType, chainId, '/cosmos/base/tendermint/v1beta1/blocks/latest', priority),
           );
         });
 

src/cron/allnodes-last-hour-downtime-check.ts

@@ -30,7 +30,7 @@ async function lastHourDowntimeP0ChainsCheck() {
         failed_requests,
         (failed_requests::FLOAT / total_requests) * 100 AS failure_rate_percent,
         CASE
-          WHEN (failed_requests::FLOAT / total_requests) * 100 > 5 THEN 'Yes'
+          WHEN (failed_requests::FLOAT / total_requests) * 100 > 10 THEN 'Yes'
           ELSE 'No'
         END AS is_downtime_hour
       FROM
@@ -148,7 +148,7 @@ async function lastHourDowntimeP1ChainsCheck() {
         failed_requests,
         (failed_requests::FLOAT / total_requests) * 100 AS failure_rate_percent,
         CASE
-          WHEN (failed_requests::FLOAT / total_requests) * 100 > 10 THEN 'Yes'
+          WHEN (failed_requests::FLOAT / total_requests) * 100 > 20 THEN 'Yes'
           ELSE 'No'
         END AS is_downtime_hour
       FROM
@@ -249,7 +249,3 @@ cron.schedule('0 * * * *', () => {
   console.log('Running scheduled task for All nodes Last One Hour Downtime P1 Chains Check...');
   lastHourDowntimeP1ChainsCheck();
 });
-
-// Initial call
-lastHourDowntimeP0ChainsCheck();
-lastHourDowntimeP1ChainsCheck();

src/cron/data-accuracy-check.ts

@@ -75,14 +75,14 @@ async function dataAccuracyCheck() {
           type: 'section',
           text: {
             type: 'mrkdwn',
-            text: '*NMS P1 Last One Hour Downtime Alert*',
+            text: '*Data Accuracy Check Alert*',
           },
         },
         {
           type: 'section',
           text: {
             type: 'mrkdwn',
-            text: 'At least one chain has experienced downtime in the last hour. Immediate attention required.',
+            text: 'Data collection less than 80 % threshold for more than 100 chains. Immediate attention required.',
           },
         },
         {
@@ -148,6 +148,3 @@ cron.schedule('0 0 * * *', () => {
   console.log('Running scheduled task to verify data accuracy...');
   dataAccuracyCheck();
 });
-
-// Initial call or you can remove it if you want to strictly follow the schedule
-dataAccuracyCheck();

src/cron/nms-api-check.ts

@@ -0,0 +1,121 @@
+import { PrismaClient } from '@prisma/client';
+import cron from 'node-cron';
+import axios from 'axios';
+import { EnvVars } from '../env-vars';
+
+const prisma = new PrismaClient();
+const slackChannelUrl = EnvVars.getSlackChannelUrlP0(); // Ensure this is configured in your environment variables
+
+async function checkApiFailuresAndAlert() {
+  const numLatestCalls: number = parseInt(process.env.NUM_LATEST_CALLS || '5', 10);
+  console.log(`Checking the last ${numLatestCalls} API calls for each chain ID and priority...`);
+
+  // Explicitly casting the query result to any[] to bypass TypeScript's strict typing.
+  const rawData = (await prisma.$queryRaw<any[]>`
+    SELECT * FROM (
+    SELECT
+      r.chain_id,
+      r."createdAt",  -- Correctly referencing with case sensitivity
+      r.type || '+' || CAST(r.priority AS TEXT) AS type_priority,  -- Using || for string concatenation
+      r.priority,  -- Ensure priority is selected if it's needed for sorting
+      r.url,
+      r."httpResponseCode",
+      r."errorMessage",
+      ROW_NUMBER() OVER (PARTITION BY r.chain_id, r.priority ORDER BY r."createdAt" DESC) as rn
+    FROM response_codes r
+    WHERE r.type = 'nms' AND (r.priority = 0 OR r.priority = 1)
+  ) sub
+  WHERE sub.rn <= 5
+  ORDER BY sub.chain_id, sub.priority, sub."createdAt" DESC;
+  `) as any[];
+
+const groupedResults = rawData.reduce((acc: { [key: string]: any[] }, item: any) => {
+  const key = `${item.chain_id}|${item.priority}`; // Changed delimiter to '|'
+  if (!acc[key]) {
+    acc[key] = [];
+  }
+  acc[key].push(item);
+  return acc;
+}, {});
+
+if (!slackChannelUrl) {
+  console.log('Slack channel URL not configured. Skipping Slack notification.');
+  return;
+}
+
+// Check if all responses are unsuccessful and send an alert
+for (const [key, details] of Object.entries(groupedResults)) {
+  if (details.every((d: any) => d.httpResponseCode !== 200)) {
+    const [chainId, priority] = key.split('|'); // Split using the new delimiter
+
+    const message = {
+      blocks: [
+        {
+          type: 'header',
+          text: {
+            type: 'plain_text',
+            text: `🚨 Alert: NMS API Failures Detected`,
+            emoji: true,
+          },
+        },
+        {
+          type: 'section',
+          text: {
+            type: 'mrkdwn',
+            text: `*Chain ID:* ${chainId}\n*Priority:* ${priority}`, // Showing Chain ID and Priority on separate lines
+          },
+        },
+        {
+          type: 'section',
+          text: {
+            type: 'mrkdwn',
+            text: `All last *${details.length} API calls* failed. Details are listed below:`,
+          },
+        },
+        ...details.map((d) => ({
+          type: 'section',
+          fields: [
+            {
+              type: 'mrkdwn',
+              text: `*Timestamp:* ${d.createdAt}`,
+            },
+            {
+              type: 'mrkdwn',
+              text: `*Type + Priority:* ${d.type_priority}`,
+            },
+            {
+              type: 'mrkdwn',
+              text: `*URL:* <${d.url}|Link>`,
+            },
+            {
+              type: 'mrkdwn',
+              text: `*Response Code:* ${d.httpResponseCode}`,
+            },
+            {
+              type: 'mrkdwn',
+              text: `*Error Message:* ${d.errorMessage || 'N/A'}`,
+            },
+          ],
+        })),
+      ],
+    };
+
+    try {
+      const response = await axios.post(slackChannelUrl, message, {
+        headers: { 'Content-Type': 'application/json' },
+      });
+      if (response.status === 200) {
+        console.log(`Alert sent to Slack for ${chainId} at priority ${priority}`);
+      } else {
+        console.error('Failed to send message to Slack:', response.data);
+      }
+    } catch (error) {
+      console.error('Failed to send message to Slack:', error);
+    }
+  }
+}
+
+}
+
+// Schedule the task to run every minute
+cron.schedule('* * * * *', checkApiFailuresAndAlert);

src/cron/nms-p1-last-hour-downtime-check.ts

@@ -31,7 +31,7 @@ async function lastHourDowntimeP0ChainsCheck() {
         failed_requests,
         (failed_requests::FLOAT / total_requests) * 100 AS failure_rate_percent,
         CASE
-          WHEN (failed_requests::FLOAT / total_requests) * 100 > 5 THEN 'Yes'
+          WHEN (failed_requests::FLOAT / total_requests) * 100 > 10 THEN 'Yes'
           ELSE 'No'
         END AS is_downtime_hour
       FROM
@@ -152,7 +152,7 @@ async function lastHourDowntimeP1ChainsCheck() {
         failed_requests,
         (failed_requests::FLOAT / total_requests) * 100 AS failure_rate_percent,
         CASE
-          WHEN (failed_requests::FLOAT / total_requests) * 100 > 10 THEN 'Yes'
+          WHEN (failed_requests::FLOAT / total_requests) * 100 > 20 THEN 'Yes'
           ELSE 'No'
         END AS is_downtime_hour
       FROM
@@ -254,7 +254,3 @@ cron.schedule('0 * * * *', () => {
   console.log('Running scheduled task for NMS Last One Hour Downtime P1 Chains Check...');
   lastHourDowntimeP1ChainsCheck();
 });
-
-// Initial call
-lastHourDowntimeP0ChainsCheck();
-lastHourDowntimeP1ChainsCheck();

src/cron/numia-last-hour-downtime-check.ts

@@ -30,7 +30,7 @@ async function lastHourDowntimeP0ChainsCheck() {
         failed_requests,
         (failed_requests::FLOAT / total_requests) * 100 AS failure_rate_percent,
         CASE
-          WHEN (failed_requests::FLOAT / total_requests) * 100 > 5 THEN 'Yes'
+          WHEN (failed_requests::FLOAT / total_requests) * 100 > 10 THEN 'Yes'
           ELSE 'No'
         END AS is_downtime_hour
       FROM
@@ -148,7 +148,7 @@ async function lastHourDowntimeP1ChainsCheck() {
         failed_requests,
         (failed_requests::FLOAT / total_requests) * 100 AS failure_rate_percent,
         CASE
-          WHEN (failed_requests::FLOAT / total_requests) * 100 > 10 THEN 'Yes'
+          WHEN (failed_requests::FLOAT / total_requests) * 100 > 20 THEN 'Yes'
           ELSE 'No'
         END AS is_downtime_hour
       FROM
@@ -249,7 +249,3 @@ cron.schedule('0 * * * *', () => {
   console.log('Running scheduled task for Numia Last One Hour Downtime P1 Chains Check...');
   lastHourDowntimeP1ChainsCheck();
 });
-
-// Initial call
-lastHourDowntimeP0ChainsCheck();
-lastHourDowntimeP1ChainsCheck();

src/env-vars.ts

@@ -50,4 +50,9 @@ export namespace EnvVars {
     const value = getEnvVar('SLACK_CHANNEL_URL');
     return isUnset(value) ? null : value;
   }
+
+  export function getSlackChannelUrlP0(): string | null {
+    const value = getEnvVar('SLACK_CHANNEL_URL_P0');
+    return isUnset(value) ? null : value;
+  }
 }