From 67ab7ca917560087fbf792d8eca1ece17b6f554e Mon Sep 17 00:00:00 2001 From: tabarra <1808295+tabarra@users.noreply.github.com> Date: Sat, 12 Oct 2024 23:19:24 -0300 Subject: [PATCH] feat(core): added diagnostic stats for server events --- core/components/Logger/handlers/server.js | 26 +++- .../StatsManager/statsUtils.test.ts | 98 ++++++++++++--- core/components/StatsManager/statsUtils.ts | 116 ++++++++++++++---- .../webroutes/diagnostics/diagnosticsFuncs.ts | 8 +- 4 files changed, 197 insertions(+), 51 deletions(-) diff --git a/core/components/Logger/handlers/server.js b/core/components/Logger/handlers/server.js index 7a21799c7..8f35b7dee 100644 --- a/core/components/Logger/handlers/server.js +++ b/core/components/Logger/handlers/server.js @@ -1,8 +1,10 @@ /* eslint-disable padded-blocks */ const modulename = 'Logger:Server'; +import { QuantileArray, estimateArrayJsonSize } from '@core/components/StatsManager/statsUtils'; import { LoggerBase } from '../LoggerBase'; import { getBootDivider } from '../loggerUtils'; import consoleFactory from '@extras/console'; +import bytes from 'bytes'; const console = consoleFactory(modulename); /* @@ -64,15 +66,34 @@ export default class ServerLogger extends LoggerBase { this.recentBuffer = []; this.recentBufferMaxSize = 32e3; + + //stats stuff + this.eventsPerMinute = new QuantileArray(24 * 60, 6 * 60); //max 1d, min 6h + this.eventsThisMinute = 0; + setInterval(() => { + this.eventsPerMinute.count(this.eventsThisMinute); + this.eventsThisMinute = 0; + }, 60_000); } /** * Returns a string with short usage stats - * TODO: calculate events per minute moving average 10 && peak */ getUsageStats() { - return `Buffer: ${this.recentBuffer.length}, lrErrors: ${this.lrErrors}`; + // Get events/min + const eventsPerMinRes = this.eventsPerMinute.resultSummary(); + const eventsPerMinStr = eventsPerMinRes.enoughData + ? eventsPerMinRes.summary + : 'LowCount'; + + //Buffer JSON size (8k min buffer, 1k samples) + const bufferJsonSizeRes = estimateArrayJsonSize(this.recentBuffer, 4e3); + const bufferJsonSizeStr = bufferJsonSizeRes.enoughData + ? `${bytes(bufferJsonSizeRes.bytesPerElement)}/e` + : 'LowCount'; + + return `Buffer: ${this.recentBuffer.length}, lrErrors: ${this.lrErrors}, mem: ${bufferJsonSizeStr}, rate: ${eventsPerMinStr}`; } @@ -109,6 +130,7 @@ export default class ServerLogger extends LoggerBase { } //Add to recent buffer + this.eventsThisMinute++; this.recentBuffer.push(eventObject); if (this.recentBuffer.length > this.recentBufferMaxSize) this.recentBuffer.shift(); diff --git a/core/components/StatsManager/statsUtils.test.ts b/core/components/StatsManager/statsUtils.test.ts index 249659297..92e5dabd2 100644 --- a/core/components/StatsManager/statsUtils.test.ts +++ b/core/components/StatsManager/statsUtils.test.ts @@ -1,5 +1,12 @@ +//@ts-nocheck import { test, expect, suite, it } from 'vitest'; -import { MultipleCounter, QuantileArray, TimeCounter } from './statsUtils'; +import { + MultipleCounter, + QuantileArray, + TimeCounter, + estimateArrayJsonSize, + isWithinMargin, +} from './statsUtils'; suite('MultipleCounter', () => { @@ -76,41 +83,53 @@ suite('QuantileArray', () => { const array = new QuantileArray(4, 2); test('min data', () => { array.count(0); - expect(array.result()).toEqual({ notEnoughData: true }); + expect(array.result()).toEqual({ enoughData: false }); }); test('zeros only', () => { array.count(0); array.count(0); array.count(0); expect(array.result()).toEqual({ + enoughData: true, count: 4, - q5: 0, - q25: 0, - q50: 0, - q75: 0, - q95: 0, + p5: 0, + p25: 0, + p50: 0, + p75: 0, + p95: 0, }); }); + const repeatedExpectedResult = { + enoughData: true, + count: 4, + p5: 0, + p25: 0, + p50: 0.5, + p75: 1, + p95: 1, + } test('calc quantile', () => { array.count(1); array.count(1); - expect(array.result()).toEqual({ - count: 4, - q5: 0, - q25: 0, - q50: 0.5, - q75: 1, - q95: 1, - }); + expect(array.result()).toEqual(repeatedExpectedResult); }); test('summary', () => { - expect(array.resultSummary()).toEqual('(4) p5:0ms/p25:0ms/p50:1ms/p75:1ms/p95:1ms'); - expect(array.resultSummary('x')).toEqual('(4) p5:0x/p25:0x/p50:1x/p75:1x/p95:1x'); + expect(array.resultSummary('ms')).toEqual({ + ...repeatedExpectedResult, + summary: '(4) p5:0ms/p25:0ms/p50:1ms/p75:1ms/p95:1ms', + }); + expect(array.resultSummary()).toEqual({ + ...repeatedExpectedResult, + summary: '(4) p5:0/p25:0/p50:1/p75:1/p95:1', + }); }); test('clear', () => { array.clear(); - expect(array.result()).toEqual({ notEnoughData: true }); - expect(array.resultSummary()).toEqual('not enough data available'); + expect(array.result()).toEqual({ enoughData: false }); + expect(array.resultSummary()).toEqual({ + enoughData: false, + summary: 'not enough data available', + }); }); }); @@ -136,3 +155,44 @@ suite('TimeCounter', async () => { expect(duration.nanoseconds / 1_000_000).toSatisfy(isCloseTo50ms); }); }); + + +suite('estimateArrayJsonSize', () => { + test('obeys minimas', () => { + const result = estimateArrayJsonSize([], 1); + expect(result).toEqual({ enoughData: false }); + + const result2 = estimateArrayJsonSize([1], 2); + expect(result2).toEqual({ enoughData: false }); + }); + + test('calculates size correctly', () => { + const array = Array.from({ length: 1000 }, (_, i) => ({ id: i, value: `value${i}` })); + const realFullSize = JSON.stringify(array).length; + const realElementSize = realFullSize / array.length; + const result = estimateArrayJsonSize(array, 100); + expect(result.enoughData).toBe(true); + expect(result.bytesTotal).toSatisfy((x: number) => isWithinMargin(x, realFullSize, 0.1)); + expect(result.bytesPerElement).toSatisfy((x: number) => isWithinMargin(x, realElementSize, 0.1)); + }); + + test('handles small arrays', () => { + const array = [{ id: 1, value: 'value1' }]; + const result = estimateArrayJsonSize(array, 0); + expect(result.enoughData).toBe(true); + expect(result.bytesTotal).toBeGreaterThan(0); + expect(result.bytesTotal).toBeLessThan(100); + expect(result.bytesPerElement).toBeGreaterThan(0); + expect(result.bytesTotal).toBeLessThan(100); + }); + + test('handles large arrays', () => { + const array = Array.from({ length: 20000 }, (_, i) => ({ id: i, value: `value${i}` })); + const realFullSize = JSON.stringify(array).length; + const realElementSize = realFullSize / array.length; + const result = estimateArrayJsonSize(array, 100); + expect(result.enoughData).toBe(true); + expect(result.bytesTotal).toSatisfy((x: number) => isWithinMargin(x, realFullSize, 0.1)); + expect(result.bytesPerElement).toSatisfy((x: number) => isWithinMargin(x, realElementSize, 0.1)); + }); +}); diff --git a/core/components/StatsManager/statsUtils.ts b/core/components/StatsManager/statsUtils.ts index 8ab8c8b74..8784d128f 100644 --- a/core/components/StatsManager/statsUtils.ts +++ b/core/components/StatsManager/statsUtils.ts @@ -59,7 +59,7 @@ export class MultipleCounter extends Map { let iterable; if (newData instanceof MultipleCounter || Array.isArray(newData)) { iterable = newData; - } else if (typeof newData === 'object' && newData !== null){ + } else if (typeof newData === 'object' && newData !== null) { iterable = Object.entries(newData); } else { throw new Error(`Invalid data type for merge`); @@ -148,16 +148,17 @@ export class QuantileArray { result(): QuantileArrayOutput { if (this.#cache.size < this.#minSize) { return { - notEnoughData: true, + enoughData: false, } } else { return { + enoughData: true, count: this.#cache.size, - q5: d3array.quantile(this.#cache.values(), 0.05)!, - q25: d3array.quantile(this.#cache.values(), 0.25)!, - q50: d3array.quantile(this.#cache.values(), 0.50)!, - q75: d3array.quantile(this.#cache.values(), 0.75)!, - q95: d3array.quantile(this.#cache.values(), 0.95)!, + p5: d3array.quantile(this.#cache.values(), 0.05)!, + p25: d3array.quantile(this.#cache.values(), 0.25)!, + p50: d3array.quantile(this.#cache.values(), 0.50)!, + p75: d3array.quantile(this.#cache.values(), 0.75)!, + p95: d3array.quantile(this.#cache.values(), 0.95)!, }; } } @@ -165,20 +166,22 @@ export class QuantileArray { /** * Returns a human readable summary of the data. */ - resultSummary(unit = 'ms') { - if (this.#cache.size < this.#minSize) { - return 'not enough data available' - } - - const data = { - p5: d3array.quantile(this.#cache.values(), 0.05)!, - p25: d3array.quantile(this.#cache.values(), 0.25)!, - p50: d3array.quantile(this.#cache.values(), 0.50)!, - p75: d3array.quantile(this.#cache.values(), 0.75)!, - p95: d3array.quantile(this.#cache.values(), 0.95)!, + resultSummary(unit = ''): QuantileArraySummary { + const result = this.result(); + if (!result.enoughData) { + return { + ...result, + summary: 'not enough data available', + }; + } + // const a = Object.entries(result) + const percentiles = (Object.entries(result) as [string, number][]) + .filter((el): el is [string, number] => el[0].startsWith('p')) + .map(([key, val]) => `${key}:${Math.ceil(val)}${unit}`); + return { + ...result, + summary: `(${this.#cache.size}) ` + percentiles.join('/'), }; - const output = Object.entries(data).map(([key, val]) => `${key}:${Math.ceil(val)}${unit}`); - return `(${this.#cache.size}) ` + output.join('/'); } toJSON() { @@ -190,16 +193,21 @@ export class QuantileArray { } } type QuantileArrayOutput = { + enoughData: true; count: number; - q5: number; - q25: number; - q50: number; - q75: number; - q95: number; + p5: number; + p25: number; + p50: number; + p75: number; + p95: number; } | { - notEnoughData: true; + enoughData: false; }; //if less than min size +type QuantileArraySummary = QuantileArrayOutput & { + summary: string, +}; + /** * Helper class to count time durations and convert them to human readable values @@ -237,3 +245,59 @@ export class TimeCounter { return this.toJSON(); } } + + +/** + * Estimates the JSON size in bytes of an array based on a simple heuristic + */ +export const estimateArrayJsonSize = (srcArray: any[], minLength: number): JsonEstimateResult => { + // Check if the buffer has enough data + if (srcArray.length <= minLength) { + return { enoughData: false }; + } + + // Determine a reasonable sample size: + // - At least 100 elements + // - Up to 10% of the buffer length + // - Capped at 1000 elements to limit CPU usage + const sourceArrayLength = srcArray.length; + const sampleSize = Math.min(1000, Math.max(100, Math.floor(sourceArrayLength * 0.1))); + const sampleArray: any[] = []; + + // Randomly sample elements from the buffer + for (let i = 0; i < sampleSize; i++) { + const randomIndex = Math.floor(Math.random() * sourceArrayLength); + sampleArray.push(srcArray[randomIndex]); + } + + // Serialize the sample to JSON + const jsonString = JSON.stringify(sampleArray); + const sampleSizeBytes = Buffer.byteLength(jsonString, 'utf-8'); // More accurate byte count + + // Estimate the total size based on the sample + const estimatedTotalBytes = (sampleSizeBytes / sampleSize) * sourceArrayLength; + const bytesPerElement = estimatedTotalBytes / sourceArrayLength; + + return { + enoughData: true, + bytesTotal: Math.round(estimatedTotalBytes), + bytesPerElement: Math.ceil(bytesPerElement), + }; +}; + +type JsonEstimateResult = { + enoughData: false; +} | { + enoughData: true; + bytesTotal: number; + bytesPerElement: number; +}; + + +/** + * Checks if a value is within a fraction margin of an expected value. + */ +export const isWithinMargin = (value: number, expectedValue: number, marginFraction: number) => { + const margin = expectedValue * marginFraction; + return Math.abs(value - expectedValue) <= margin; +} diff --git a/core/webroutes/diagnostics/diagnosticsFuncs.ts b/core/webroutes/diagnostics/diagnosticsFuncs.ts index 0a634418f..13539ef42 100644 --- a/core/webroutes/diagnostics/diagnosticsFuncs.ts +++ b/core/webroutes/diagnostics/diagnosticsFuncs.ts @@ -277,10 +277,10 @@ export const getTxAdminData = async (txAdmin: TxAdmin) => { units: ['d', 'h', 'm'], }; - const banCheckTime = txAdmin.statsManager.txRuntime.banCheckTime.resultSummary(); - const whitelistCheckTime = txAdmin.statsManager.txRuntime.whitelistCheckTime.resultSummary(); - const playersTableSearchTime = txAdmin.statsManager.txRuntime.playersTableSearchTime.resultSummary(); - const historyTableSearchTime = txAdmin.statsManager.txRuntime.historyTableSearchTime.resultSummary(); + const banCheckTime = txAdmin.statsManager.txRuntime.banCheckTime.resultSummary('ms'); + const whitelistCheckTime = txAdmin.statsManager.txRuntime.whitelistCheckTime.resultSummary('ms'); + const playersTableSearchTime = txAdmin.statsManager.txRuntime.playersTableSearchTime.resultSummary('ms'); + const historyTableSearchTime = txAdmin.statsManager.txRuntime.historyTableSearchTime.resultSummary('ms'); const memoryUsage = getHeapStatistics(); return {