RED-195: Add support for max queue time on node loads page

README.md

@@ -5,3 +5,26 @@
 If you're daring and want to cut and push a release to npm, you can do so
 solely by running `npm run release`. But I warn you - with great power, comes
 great responsibility ;)
+
+## How to run a custom monitor client branch (dev) with a local network.
+
+1. go to the monitor client folder:
+
+-   run npm ci
+-   run npm link
+-   run npm run prepare
+
+2. go to the monitor server folder:
+
+-   run npm ci
+-   run npm link @shardus/monitor-client
+-   run npm run prepare
+
+3. Go to your running (if running) shardeum-server folder
+
+-   shardus pm2 list // find the index of monitor server usually 1
+-   shardus pm2 stop <index> // index usually 1
+
+4. go to monitor server folder
+
+-   npm run start (edited)

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@shardus/monitor-client",
-    "version": "2.6.0",
+    "version": "2.8.0",
     "description": "",
     "main": "entry.js",
     "engines": {

public/large-network.js

@@ -53,6 +53,14 @@
                 nodeLoads: [],
                 sortKey: 'ip',
                 sortAsc: true,
+                oosFilter: 'raw',
+                oosSummary: {
+                    E: { count: 0, nodes: 0 },
+                    CE: { count: 0, nodes: 0 },
+                    C: { count: 0, nodes: 0 }
+                },
+                isRecentActiveCycles: 4,
+                recentRuntimeSyncMap: new Map(),
             }
         },
         async mounted() {
@@ -69,6 +77,11 @@
                 })
             },
         },
+        watch: {
+            oosFilter() {
+                this.refreshNodeColors()
+            }
+        },
         methods: {
             calculateNetworkPosition(nodeId) {
                 let spread = 4
@@ -173,18 +186,63 @@
                 if (!node.cycleMarker) return '#fcbf49' // syncing node
                 let color = '#000000'
                 if (this.colorMode === 'state') {
-                    color = node.isDataSynced ? '#80ED99' : '#FF2EFF'
+                    if (this.oosFilter === 'raw') {
+                        color = node.isDataSynced ? '#80ED99' : '#FF2EFF' // green for data synced, magenta for data not synced
+                    } else {
+                        const oos = this.isUnexpectedOOS(node, this.oosFilter === 'smart-c')
+                        if (oos.total > 0) {
+                            color = '#FF2EFF' // Magenta for unexpected OOS
+                        } else {
+                            const nodeRadixes = node.radixes || []
+                            // if smart-c, only consider radixes in consensus range
+                            // if smart-any, consider all radixes
+                            const recentOOS = nodeRadixes
+                                .filter(radix => {
+                                    if (this.oosFilter === 'smart-c') {
+                                        return radix.inConsensusRange; // Only C and CE for 'smart-c'
+                                    }
+                                    return true; // All radixes for 'smart-any'
+                                })
+                                .map(radix => {
+                                    const uniqueKey = `${node.nodeId}-${radix.radix}`
+                                    return this.recentRuntimeSyncMap.get(uniqueKey) || 0
+                                })
+
+                            if (recentOOS.some(oosCycle => oosCycle > 0)) {
+                                const mostRecentOOS = Math.max(...recentOOS);
+                                const cyclesSinceOOS = this.networkStatus.counter - mostRecentOOS;
+
+                                if (cyclesSinceOOS <= 1) {
+                                    color = '#00FFFF' // Bright cyan for very recent OOS (0-1 cycles ago)
+                                } else if (cyclesSinceOOS <= 2) {
+                                    color = '#00CCCC' // Medium cyan (1-2 cycles ago)
+                                } else if (cyclesSinceOOS <= 3) {
+                                    color = '#009999' // Darker cyan (2-3 cycles ago)
+                                } else if (cyclesSinceOOS <= 4) {
+                                    color = '#006666' // Darkest cyan (3-4 cycles ago)
+                                } else {
+                                    color = '#80ED99' // Return to green after 4 cycles
+                                }
+                            } else {
+                                color = '#80ED99' // green for no recent OOS
+                            }
+                        }
+                    }
                     if (node.crashed && !node.isRefuted) {
                         color = '#FF2442'
                     }
-                } else if (this.colorMode === 'marker') color = `#${node.cycleMarker.substr(0, 6)}`
-                else if (this.colorMode === 'nodelist') color = `#${node.nodelistHash.substr(0, 6)}`
+                } else if (this.colorMode === 'marker') {
+                    color = `#${node.cycleMarker.substr(0, 6)}`
+                } else if (this.colorMode === 'nodelist') {
+                    color = `#${node.nodelistHash.substr(0, 6)}`
+                }
                 return color
             },
             onColorModeChange(event) {
                 if (event.target.value === this.colorMode) return
                 this.colorMode = event.target.value
                 this.changeNodeColor()
+                this.refreshNodeColors()
             },
             async fetchChanges() {
                 let res = await requestWithToken(
@@ -243,7 +301,18 @@
                     queueLength.push(node.queueLength)
                     totalQueueTime += node.txTimeInQueue
                     queueTime.push(node.txTimeInQueue)
-
+                    const result = node.lastInSyncResult
+                    this.networkStatus.counter = node.cycleCounter
+
+                    for (let radix of result?.radixes || []) {
+                        const recentRuntimeSyncCycle = radix.recentRuntimeSyncCycle || -1
+                        const uniqueKey = `${nodeId}-${radix.radix}`
+                        if (recentRuntimeSyncCycle !== -1) {
+                            this.recentRuntimeSyncMap.set(uniqueKey, recentRuntimeSyncCycle)
+                        }
+                    }
+                    node.radixes = result?.radixes || []
+
                     this.nodeLoads.push({
                         id: nodeId,
                         ip: node.nodeIpInfo.externalIp,
@@ -252,6 +321,14 @@
                         loadExternal: node.currentLoad.nodeLoad.external,
                         queueLength: node.queueLength,
                         queueTime: node.txTimeInQueue,
+                        inSync: result?.insync,
+                        total: result?.stats.total,
+                        good: result?.stats.good,
+                        bad: result?.stats.bad,
+                        radixes: result?.radixes,
+                        stillNeedsInitialPatchPostActive: node.stillNeedsInitialPatchPostActive,
+                        cycleFinishedSyncing: node.cycleFinishedSyncing,
+                        recentRuntimeSync: result?.radixes.some((r) => r.recentRuntimeSync),
                     })
                 }
 
@@ -433,6 +510,9 @@
                     updatedNodes = Object.values(updatedNodesMap)
                     G.visNodes.update(updatedNodes)
 
+                    // update oos summary
+                    this.oosSummary = this.calculateOOSSummary()
+
                     // delete removed nodes
                     await this.deleteRemovedNodes()
 
@@ -442,6 +522,8 @@
                         this.deleteCrashedNodes(crashedNodesToRemove)
                     }
 
+                    this.refreshNodeColors()
+
                     // G.lastUpdatedTimestamp = Date.now()
                     if (this.shouldChangeNodesSize()) this.changeNodesSize()
                 } catch (e) {
@@ -708,6 +790,82 @@
                     nodeDimensions: { width, height },
                 }
             },
+            // Check if node is in unexpected out of sync state
+            isUnexpectedOOS(node, CAndCEOnly = false) {
+                const currentCounter = this.networkStatus.counter
+                let CUnexpectedOOSCount = 0
+                let EUnexpectedOOSCount = 0
+                let CEUnexpectedOOSCount = 0
+                // Check if node.radixes exists and is iterable
+                if (node.radixes && typeof node.radixes[Symbol.iterator] === 'function') {
+                    for (let radix of node.radixes) {
+                        if (CAndCEOnly && radix.inEdgeRange) continue
+                        if (!radix.insync) {
+                            const recentlyActive =
+                                currentCounter - node.cycleFinishedSyncing <= this.isRecentActiveCycles
+                            const hasRecentSync = radix.recentRuntimeSync
+
+                            if (!recentlyActive && !hasRecentSync) {
+                                if (radix.inConsensusRange && radix.isEdgeRange) {
+                                    CEUnexpectedOOSCount++
+                                } else if (radix.inConsensusRange) {
+                                    CUnexpectedOOSCount++
+                                } else if (radix.inEdgeRange) {
+                                    EUnexpectedOOSCount++
+                                }
+                            }
+                        }
+                    }
+                } else {
+                    console.warn(`Node ${node.id || 'unknown'} does not have a valid radixes property`)
+                }
+
+                return {
+                    total: CUnexpectedOOSCount + EUnexpectedOOSCount + CEUnexpectedOOSCount,
+                    C: CUnexpectedOOSCount,
+                    E: EUnexpectedOOSCount,
+                    CE: CEUnexpectedOOSCount,
+                }
+            },
+
+            // Calculate summary of unexpected out of sync nodes
+            calculateOOSSummary() {
+                let summary = {
+                    E: { count: 0, nodes: 0 },
+                    CE: { count: 0, nodes: 0 },
+                    C: { count: 0, nodes: 0 }
+                };
+
+                for (let nodeId in G.nodes.active) {
+                    let node = G.nodes.active[nodeId];
+                    let oos = this.isUnexpectedOOS(node, this.oosFilter === 'smart-c');
+
+                    if (oos.E > 0) {
+                        summary.E.count += oos.E;
+                        summary.E.nodes++;
+                    }
+                    if (oos.CE > 0) {
+                        summary.CE.count += oos.CE;
+                        summary.CE.nodes++;
+                    }
+                    if (oos.C > 0) {
+                        summary.C.count += oos.C;
+                        summary.C.nodes++;
+                    }
+                }
+
+                return summary;
+            },
+            // update when filter changes
+            refreshNodeColors() {
+                let updatedNodes = []
+                for (let nodeId in G.nodes.active) {
+                    let node = G.nodes.active[nodeId]
+                    let updatedVisNode = this.getUpdatedVisNode(nodeId, node)
+                    updatedNodes.push(updatedVisNode)
+                }
+                G.visNodes.update(updatedNodes)
+            },
 
             async start() {
                 let res = await requestWithToken(`${monitorServerUrl}/report`)

public/node-loads.js

@@ -0,0 +1,75 @@
+(function main() {
+  const G = {}
+  loadToken(G)
+  G.monitorServerUrl = monitorServerUrl || `https://127.0.0.1:3000/api`
+  G.REFRESH_TIME = 10000
+
+  new Vue({
+      el: '#app',
+      data() {
+          return {
+              nodeLoads: [],
+              sortKey: 'ip',
+              sortAsc: true,
+          }
+      },
+      computed: {
+          sortedNodes() {
+              return this.nodeLoads.sort((a, b) => {
+                  let modifier = this.sortAsc ? 1 : -1
+                  if (a[this.sortKey] < b[this.sortKey]) return -1 * modifier
+                  if (a[this.sortKey] > b[this.sortKey]) return 1 * modifier
+                  return 0
+              })
+          },
+      },
+      methods: {
+          sortTable(key) {
+              if (this.sortKey === key) {
+                  this.sortAsc = !this.sortAsc
+              } else {
+                  this.sortKey = key
+                  this.sortAsc = true
+              }
+          },
+          async fetchChanges() {
+              let res = await requestWithToken(
+                  `${G.monitorServerUrl}/report?timestamp=${G.lastUpdatedTimestamp}`
+              )
+              return res.data
+          },
+          updateNetworkStatus(report) {
+              this.nodeLoads = []
+              for (let nodeId in report.nodes.active) {
+                  const node = report.nodes.active[nodeId]
+                  this.nodeLoads.push({
+                      id: nodeId,
+                      ip: node.nodeIpInfo.externalIp,
+                      port: node.nodeIpInfo.externalPort,
+                      loadInternal: node.currentLoad.nodeLoad.internal.toFixed(3),
+                      loadExternal: node.currentLoad.nodeLoad.external.toFixed(3),
+                      queueLength: node.queueLength,
+                      avgQueueTime: node.txTimeInQueue.toFixed(3),
+                      maxQueueTime: node.maxTxTimeInQueue.toFixed(3),
+                  })
+              }
+          },
+          async updateNodes() {
+              try {
+                  let changes = await this.fetchChanges()
+                  this.updateNetworkStatus(changes)
+              } catch (e) {
+                  console.log('Error while trying to update nodes.', e)
+              }
+          },
+          start() {
+              this.updateNodes()
+              setInterval(this.updateNodes, G.REFRESH_TIME)
+          },
+      },
+      mounted() {
+          console.log('Mounted')
+          this.start()
+      },
+  })
+})()