Merge pull request #34 from ara-klaytn/L1finalisation

Added L1 commitment latency and slack messages upon failure

aptos-tx-latency-measurement/.env.template

@@ -17,3 +17,4 @@ AWS_PROFILE=default
 # Please set this value to the desired interval in milliseconds. For example, to send
 # transactions 1 hour, you can set SEND_TX_INTERVAL as 60 * 60 * 1000.
 SEND_TX_INTERVAL=
+COIN_GECKO_API_KEY=

aptos-tx-latency-measurement/sendtx_aptos.js

@@ -181,8 +181,9 @@ async function sendTx() {
     const balance = await coinClient.checkBalance(account);
 
     if (balance < parseFloat(process.env.BALANCE_ALERT_CONDITION_IN_APTOS)) {
+      const now = new Date();
       sendSlackMsg(
-        `Current balance of <${process.env.SCOPE_URL}/address/${address}|${address}> is less than ${process.env.BALANCE_ALERT_CONDITION_IN_APTOS} APTOS! balance=${balance} APTOS`
+        `${now}, Current balance of <${process.env.SCOPE_URL}/address/${address}|${address}> is less than ${process.env.BALANCE_ALERT_CONDITION_IN_APTOS} APTOS! balance=${balance} APTOS`
       );
     }
 
@@ -212,14 +213,12 @@ async function sendTx() {
     data.chainId = process.env.CHAIN_ID;
 
     var APTOStoUSD;
-    await CoinGeckoClient.simple
-      .price({
-        ids: ["aptos"],
-        vs_currencies: ["usd"],
-      })
-      .then((response) => {
-        APTOStoUSD = response.data["aptos"]["usd"];
-      });
+
+    await axios.get(`https://api.coingecko.com/api/v3/simple/price?ids=aptos&vs_currencies=usd&x_cg_demo_api_key=${process.env.COIN_GECKO_API_KEY}`)
+    .then(response => {
+      APTOStoUSD = response.data["aptos"].usd;
+    });
+
     const transactionDetail = await client.getTransactionByHash(txnHash);
     const gasUsed = transactionDetail.gas_used;
     const gasUnitPrice = transactionDetail.gas_unit_price;
@@ -228,14 +227,17 @@ async function sendTx() {
     data.txFeeInUSD = APTOStoUSD * data.txFee;
     // console.log(`${data.executedAt},${data.chainId},${data.txhash},${data.startTime},${data.endTime},${data.latency},${data.txFee},${data.txFeeInUSD},${data.resourceUsedOfLatestBlock},${data.numOfTxInLatestBlock},${data.pingTime},${data.error}`)
   } catch (err) {
-    sendSlackMsg("failed to execute, " + err.toString());
+
+     const now = new Date();
+    sendSlackMsg(`${now}, failed to execute aptos, ${err.toString()}`);
     console.log("failed to execute.", err.toString());
     data.error = err.toString();
     console.log(`${data.executedAt},${data.chainId},${data.txhash},${data.startTime},${data.endTime},${data.latency},${data.txFee},${data.txFeeInUSD},${data.resourceUsedOfLatestBlock},${data.numOfTxInLatestBlock},${data.pingTime},${data.error}`)
   }
   try {
     await uploadChoice(data);
   } catch (err) {
+    sendSlackMsg(`${now}, failed to upload aptos, ${err.toString()}`);
     console.log(
       `failed to ${process.env.UPLOAD_METHOD === "AWS" ? "s3" : "gcs"}.upload!! Printing instead!`,
       err.toString()
@@ -263,6 +265,7 @@ async function main() {
     console.log(`sending tx...`);
     sendTx();
   }, interval);
+  sendTx();
 }
 loadConfig();
 main();

arbitrium-tx-latency-measurement/.env.template

@@ -5,14 +5,21 @@ S3_BUCKET=
 GCP_PROJECT_ID=
 GCP_KEY_FILE_PATH=
 GCP_BUCKET=
+GCP_PROJECT_ID_L1=
+GCP_KEY_FILE_PATH_L1=
+GCP_BUCKET_L1=
 SLACK_CHANNEL=SLACK_CHANNEL
 SLACK_API_URL=https://slack.com/api/chat.postMessage
 SLACK_AUTH=xoxb-SLACK_AUTH_TOKEN
 BALANCE_ALERT_CONDITION_IN_ARB=0.0001
 SCOPE_URL=https://goerli.arbiscan.io/
 AWS_PROFILE=default
 
+# L1FINALITYSCRAPERURL docker example http://localhost:8080
+L1FINALITYSCRAPERURL=
+
 # The SEND_TX_INTERVAL specifies the interval in milliseconds for sending transactions.
 # Please set this value to the desired interval in milliseconds. For example, to send
 # transactions every 1 hour, you can set SEND_TX_INTERVAL as 60 * 60 * 1000.
-SEND_TX_INTERVAL=
+SEND_TX_INTERVAL=
+COIN_GECKO_API_KEY=

arbitrium-tx-latency-measurement/package.json

@@ -8,13 +8,15 @@
   },
   "author": "",
   "license": "ISC",
+  "type": "module",
   "dependencies": {
     "@arbitrum/sdk": "^3.1.11",
     "@google-cloud/storage": "^6.0.0",
     "aws-sdk": "^2.1101.0",
     "axios": "0.26.1",
     "coingecko-api": "1.0.10",
     "dotenv": "16.0.0",
+    "lowdb": "^6.1.1",
     "moment": "2.29.3",
     "parquetjs-lite": "0.8.7",
     "web3": "^4.1.2"

arbitrium-tx-latency-measurement/sendtx_arbitrium.js

@@ -1,14 +1,16 @@
 // Arbitrium PoS transaction latency measurement.
 
-const { Web3 } = require("web3");
-const fs = require("fs");
-const AWS = require("aws-sdk");
-const parquet = require("parquetjs-lite");
-const moment = require("moment");
-const axios = require("axios");
-const CoinGecko = require("coingecko-api");
-const { Storage } = require("@google-cloud/storage");
-require("dotenv").config();
+import "dotenv/config";
+
+import { Web3 } from "web3";
+import fs from "fs";
+import AWS from "aws-sdk";
+import parquet from "parquetjs-lite";
+import moment from "moment";
+import axios from "axios";
+import CoinGecko from "coingecko-api";
+import { Storage } from "@google-cloud/storage";
+import { JSONPreset } from "lowdb/node";
 
 let rpc = process.env.PUBLIC_RPC_URL;
 const provider = new Web3.providers.HttpProvider(rpc);
@@ -117,6 +119,36 @@ async function uploadToGCS(data) {
   fs.unlinkSync(filename);
 }
 
+async function uploadToGCSL1(data) {
+  if (
+    process.env.GCP_PROJECT_ID_L1 === "" ||
+    process.env.GCP_KEY_FILE_PATH_L1 === "" ||
+    process.env.GCP_BUCKET_L1 === ""
+  ) {
+    throw "undefined parameters";
+  }
+
+  const storage = new Storage({
+    projectId: process.env.GCP_PROJECT_ID_L1,
+    keyFilename: process.env.GCP_KEY_FILE_PATH_L1,
+  });
+
+  const filename = await makeParquetFile(data);
+  const destFileName = `tx-latency-measurement/arbitriuml1/${filename}`;
+
+  async function uploadFile() {
+    const options = {
+      destination: destFileName,
+    };
+
+    await storage.bucket(process.env.GCP_BUCKET_L1).upload(filename, options);
+    console.log(`${filename} uploaded to ${process.env.GCP_BUCKET_L1}`);
+  }
+
+  await uploadFile().catch(console.error);
+  fs.unlinkSync(filename);
+}
+
 async function uploadChoice(data) {
   if (process.env.UPLOAD_METHOD === "AWS") {
     await uploadToS3(data);
@@ -148,10 +180,10 @@ async function sendTx() {
     const signer = web3.eth.accounts.privateKeyToAccount(privateKey);
     const balance = Number(await web3.eth.getBalance(signer.address)) * 10 ** -18; //in wei
 
-    console.log(`Current balance of ${signer.address} is ${balance} ARB`);
     if (balance < parseFloat(process.env.BALANCE_ALERT_CONDITION_IN_ARB)) {
+      const now = new Date();
       sendSlackMsg(
-        `Current balance of <${process.env.SCOPE_URL}/address/${signer.address}|${signer.address}> is less than ${process.env.BALANCE_ALERT_CONDITION_IN_ARB} ARB! balance=${balance} ARB`
+        `${now}, Current balance of <${process.env.SCOPE_URL}/address/${signer.address}|${signer.address}> is less than ${process.env.BALANCE_ALERT_CONDITION_IN_ARB} ARB! balance=${balance} ARB`
       );
     }
 
@@ -210,33 +242,103 @@ async function sendTx() {
       })
       .catch(console.error);
 
+    const db = await JSONPreset("db.json", { posts: [] });
+    db.data.posts.push({
+      l2TxHash: data.txhash,
+      createdAt: data.executedAt,
+      status: "pending",
+      l1CommitTiming: null,
+    });
+    db.write();
+
     // Calculate Transaction Fee and Get Tx Fee in USD
     var ARBtoUSD;
-    await CoinGeckoClient.simple
-      .price({
-        ids: ["arbitrum"],
-        vs_currencies: ["usd"],
-      })
-      .then((response) => {
-        ARBtoUSD = response.data["arbitrum"]["usd"];
-      });
+    await axios.get(`https://api.coingecko.com/api/v3/simple/price?ids=arbitrum&vs_currencies=usd&x_cg_demo_api_key=${process.env.COIN_GECKO_API_KEY}`)
+    .then(response => {
+      ARBtoUSD = response.data["arbitrum"].usd;
+    });
     data.txFeeInUSD = data.txFee * ARBtoUSD;
 
     // console.log(`${data.executedAt},${data.chainId},${data.txhash},${data.startTime},${data.endTime},${data.latency},${data.txFee},${data.txFeeInUSD},${data.resourceUsedOfLatestBlock},${data.numOfTxInLatestBlock},${data.pingTime},${data.error}`)
   } catch (err) {
+     const now = new Date();
+    sendSlackMsg(`${now}, failed to execute arbitrum, ${err.toString()}`);
     console.log("failed to execute.", err.toString());
     data.error = err.toString();
     // console.log(`${data.executedAt},${data.chainId},${data.txhash},${data.startTime},${data.endTime},${data.latency},${data.txFee},${data.txFeeInUSD},${data.resourceUsedOfLatestBlock},${data.numOfTxInLatestBlock},${data.pingTime},${data.error}`)
   }
   try {
     await uploadChoice(data);
   } catch (err) {
+    sendSlackMsg(`${now}, failed to upload arbitrium, ${err.toString()}`);
+
     console.log(
       `failed to ${process.env.UPLOAD_METHOD === "AWS" ? "s3" : "gcs"}.upload!! Printing instead!`,
       err.toString()
     );
-    console.log(JSON.stringify(data));
-    console.log("=======================================================");
+  }
+}
+
+async function l1Checker() {
+  await new Promise((resolve) => setTimeout(resolve, 1000 * 60 * 2));
+  const db = await JSONPreset("db.json", { posts: [] });
+  for (const post of db.data.posts) {
+    console.log(post.l2TxHash);
+    const currentTimestamp = new Date().getTime();
+    const fortyFiveMinutesAgoTimestamp = currentTimestamp - 1000 * 60 * 45;
+
+    if (post.status === "pending" && post.createdAt < fortyFiveMinutesAgoTimestamp) {
+      console.log(post.l2TxHash, "is pending");
+      await l1commitmentprocess(db, post.l2TxHash, post.createdAt);
+    }
+  }
+  await db.write();
+}
+
+async function l1commitmentprocess(db, hash, createdAt) {
+
+  var data = {
+    executedAt: new Date().getTime(),
+    txhash: "",
+    startTime: 0,
+    endTime: 0,
+    chainId: 0,
+    latency: 0,
+    error: "",
+    txFee: 0.0,
+    txFeeInUSD: 0.0,
+    resourceUsedOfLatestBlock: 0,
+    numOfTxInLatestBlock: 0,
+    pingTime: 0,
+  };
+
+  const response = await fetch(`${process.env.L1FINALITYSCRAPERURL}/root_end?from_chain=42161&hash=${hash}`);
+  if (!response.ok) {
+    const postIndex = db.data.posts.findIndex((post) => post.l2TxHash === hash);
+    if (postIndex !== -1) {
+      console.log("L1 tx hash not found");
+      db.data.posts[postIndex].status = "failed";
+      sendSlackMsg(`L1 tx hash not found for ${hash} in Arbitrium!`);
+      return null;
+    } else {
+      sendSlackMsg(`l2 ${hash} not found in Arbitrium!`);
+      return Error("l2TxHash not found.");
+    }
+  }
+  const data = await response.json();
+  const finalityTiming = parseInt(data.root_end, 10);
+  const timeTaken = finalityTiming - createdAt;
+
+  const postIndex = db.data.posts.findIndex((post) => post.l2TxHash === hash);
+  if (postIndex !== -1) {
+    db.data.posts[postIndex].l1CommitTiming = timeTaken;
+    db.data.posts[postIndex].status = "success";
+    data.latency = timeTaken;
+    data.hash = hash;
+    uploadToGCSL1(data)
+  } else {
+    sendL1FailedSlackMsg(`l2 ${hash} not found! in Arbitrium!`);
+    return Error("l2TxHash not found.");
   }
 }
 
@@ -258,7 +360,9 @@ async function main() {
   const interval = eval(process.env.SEND_TX_INTERVAL);
   setInterval(() => {
     sendTx();
+    l1Checker();
   }, interval);
+  sendTx();
 }
 
 main();

avalanche-tx-latency-measurement/.env.template

@@ -16,4 +16,5 @@ AVALANCHE_HOST=api.avax-test.network
 # The SEND_TX_INTERVAL specifies the interval in milliseconds for sending transactions.
 # Please set this value to the desired interval in milliseconds. For example, to send
 # transactions every 1 hour, you can set SEND_TX_INTERVAL as 60 * 60 * 1000.
-SEND_TX_INTERVAL=
+SEND_TX_INTERVAL=
+COIN_GECKO_API_KEY=

avalanche-tx-latency-measurement/sendtx_avalanche.js

@@ -163,7 +163,8 @@ const sendAvax = async (amount, to, maxFeePerGas = undefined, maxPriorityFeePerG
         const balance = (await HTTPSProvider.getBalance(address)) * (10**(-18))// getAssetBalance
         if(balance < parseFloat(process.env.BALANCE_ALERT_CONDITION_IN_AVAX))
         {
-            sendSlackMsg(`Current balance of <${process.env.SCOPE_URL}/address/${address}|${address}> is less than ${process.env.BALANCE_ALERT_CONDITION_IN_AVAX} AVAX! balance=${balance} AVAX`)
+            const now = new Date();
+            sendSlackMsg(`${now}, Current balance of <${process.env.SCOPE_URL}/address/${address}|${address}> is less than ${process.env.BALANCE_ALERT_CONDITION_IN_AVAX} AVAX! balance=${balance} AVAX`)
         }
 
         const latestNonce = await HTTPSProvider.getTransactionCount(address);
@@ -221,23 +222,26 @@ const sendAvax = async (amount, to, maxFeePerGas = undefined, maxPriorityFeePerG
 
         // Get tx Fee and tx Fee in USD
         var AVAXtoUSD;
-        await CoinGeckoClient.simple.price({
-            ids: ["avalanche-2"],
-            vs_currencies: ["usd"]
-        }).then((response)=>{
-            AVAXtoUSD= response.data["avalanche-2"]["usd"]
-        })
+
+        await axios.get(`https://api.coingecko.com/api/v3/simple/price?ids=avalanche-2&vs_currencies=usd&x_cg_demo_api_key=${process.env.COIN_GECKO_API_KEY}`)
+        .then(response => {
+            AVAXtoUSD = response.data["avalanche-2"].usd;
+        });
+
         data.txFee = ethers.utils.formatEther(signature.effectiveGasPrice) * signature.gasUsed;
         data.txFeeInUSD = data.txFee  * AVAXtoUSD;
         // console.log(`${data.executedAt},${data.chainId},${data.txhash},${data.startTime},${data.endTime},${data.latency},${data.txFee},${data.txFeeInUSD},${data.resourceUsedOfLatestBlock},${data.numOfTxInLatestBlock},${data.pingTime},${data.error}`)
     } catch(err){
+         const now = new Date();
+    sendSlackMsg(`${now}, failed to execute avalanche, ${err.toString()}`);
         console.log("failed to execute.", err.toString())
         data.error = err.toString()
         // console.log(`${data.executedAt},${data.chainId},${data.txhash},${data.startTime},${data.endTime},${data.latency},${data.txFee},${data.txFeeInUSD},${data.resourceUsedOfLatestBlock},${data.numOfTxInLatestBlock},${data.pingTime},${data.error}`)
     }
     try{
         await uploadChoice(data)
     } catch(err){
+        sendSlackMsg(`failed to upload avalanche, ${err.toString()}`);
         console.log(`failed to ${process.env.UPLOAD_METHOD === 'AWS'? 's3': 'gcs'}.upload!! Printing instead!`, err.toString())
         console.log(JSON.stringify(data))
     }
@@ -264,6 +268,7 @@ async function main(){
         setInterval(()=>{
         sendAvax("0.0", address);
     }, interval)
+    sendAvax("0.0", address);
 }
 
 main();

bnb-tx-latency-measurement/.env.template

@@ -14,4 +14,5 @@ SLACK_AUTH=xoxb-SLACK_AUTH_TOKEN
 # The SEND_TX_INTERVAL specifies the interval in milliseconds for sending transactions.
 # Please set this value to the desired interval in milliseconds. For example, to send
 # transactions every 1 hour, you can set SEND_TX_INTERVAL as 60 * 60 * 1000.
-SEND_TX_INTERVAL=
+SEND_TX_INTERVAL=
+COIN_GECKO_API_KEY=