From 0b9791eb0232108f8a8f2a0619b7e8ce1cef6f01 Mon Sep 17 00:00:00 2001 From: Jakob Getz Date: Tue, 6 Feb 2024 14:12:07 +0900 Subject: [PATCH] added round trip time metric to test framework --- src/performance.cts | 8 +++--- tests/run-tests.cts | 63 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/performance.cts b/src/performance.cts index f5d6f8a9..3571685d 100644 --- a/src/performance.cts +++ b/src/performance.cts @@ -2,22 +2,22 @@ import fs from 'fs/promises' import { rmSafe } from '../tests/test-utils.cjs' type PerformanceInfo = { - phase: 'record' | 'replay-generation' | 'replay', + phase: 'record' | 'replay-generation' | 'replay' | 'all', description: string } -export type StopMeasure = () => void +export type StopMeasure = () => PerformanceMeasure export function createMeasure(name: string, detail: PerformanceInfo): StopMeasure { const start = name + 'start' const end = name + '_end' performance.mark(start) return () => { performance.mark(end) - performance.measure(name, { start, end, detail }) + return performance.measure(name, { start, end, detail }) } } -type Type = 'manual-run' | 'offline-auto-test' | 'online-auto-test' +type Type = 'manual-run' | 'offline-auto-test' | 'online-auto-test' | 'node-auto-test' export async function initPerformance(app: string, type: Type, filePath?: string) { const timeStamp = Date.now() const dBPath = 'performance.db.ndjson' diff --git a/tests/run-tests.cts b/tests/run-tests.cts index 0bc795c2..a196eb61 100644 --- a/tests/run-tests.cts +++ b/tests/run-tests.cts @@ -14,6 +14,7 @@ import Analyser, { AnalysisResult } from '../src/analyser.cjs' import commandLineArgs from 'command-line-args' import { initPerformance } from '../src/performance.cjs' import { generateJavascript } from '../src/js-generator.cjs' +import { createMeasure } from '../src/performance.cjs' let extended = false @@ -54,6 +55,7 @@ async function runNodeTest(name: string, options): Promise { // 1. Instrument with Wasabi !!Please use the newest version const indexRsPath = path.join(testPath, 'index.rs') const indexCPath = path.join(testPath, 'index.c') + const p_roundTrip = createMeasure('round-trip time', { description: 'The time it takes to start the browser instance, load the webpage, record the interaction, download the data, and generate the replay', phase: 'all' }) if (fss.existsSync(indexRsPath)) { cp.execSync(`rustc --crate-type cdylib ${indexRsPath} --target wasm32-unknown-unknown --crate-type cdylib -o ${wasmPath}`, { stdio: 'ignore' }) cp.execSync(`wasm2wat ${wasmPath} -o ${watPath}`) @@ -111,14 +113,14 @@ async function runNodeTest(name: string, options): Promise { execSync(`./crates/target/debug/replay_gen ${tracePath} ${wasmPath} ${replayWasmPath}`); // we validate and early return as for single wasm accuracy test doesn't make sense execSync(`wasm-validate ${replayWasmPath}`) - return { testPath, success: true } + return { testPath, roundTripTime: p_roundTrip().duration, success: true } } } - await delay(0) // WTF why do I need this WHAT THE FUCK } catch (e: any) { return { testPath, success: false, reason: e.stack } } + let roundTripTime = p_roundTrip().duration // 4. Execute replay and generate trace and compare let replayTracer = new Tracer(eval(js + `\nWasabi`), { extended }) @@ -128,12 +130,13 @@ async function runNodeTest(name: string, options): Promise { replayTracer.init() replayBinary.replay(wasm) } catch (e: any) { - return { testPath, success: false, reason: e.stack } + return { testPath, roundTripTime, success: false, reason: e.stack } } let replayTraceString = replayTracer.getResult().toString(); await fs.writeFile(replayTracePath, replayTraceString); const result = compareResults(testPath, traceString, replayTraceString) + result.roundTripTime = roundTripTime if (result.success === false) { return result } @@ -141,6 +144,12 @@ async function runNodeTest(name: string, options): Promise { return result } +function writeSummary(type: string, testCount: number, successfull: number, roundTripTimes: DOMHighResTimeStamp[]) { + const fail = testCount - successfull + const avgRoundTripTime = roundTripTimes.reduce((p, c) => p + c) / roundTripTimes.length + console.log(`finished running ${testCount} ${type} testcases. Pass: ${successfull}, Fail: ${fail}, FailRate: ${fail / testCount * 100}%, Avg time: ${avgRoundTripTime}`); +} + async function runNodeTests(names: string[], options) { if (names.length > 0) { console.log('==============') @@ -161,11 +170,23 @@ async function runNodeTests(names: string[], options) { 'table-exp-host-add-friend', ] names = names.filter((n) => !filter.includes(n)) + let successfull = 0; // names = ["mem-imp-host-grow"] + let roundTripTimes = [] for (let name of names) { + const testPath = path.join(process.cwd(), 'tests', 'node', name) + const cleanUpPerformance = await initPerformance(name, 'node-auto-test', path.join(testPath, 'performance.ndjson')) const report = await runNodeTest(name, options) + cleanUpPerformance() await writeReport(name, report) + if (report.success === true) { + successfull++ + } + if (report.roundTripTime !== undefined) { + roundTripTimes.push(report.roundTripTime) + } } + writeSummary('node', names.length, successfull, roundTripTimes); } function compareResults(testPath: string, traceString: string, replayTraceString: string): TestReport { @@ -196,6 +217,8 @@ async function runOnlineTests(names: string[], options) { 'image-convolute', // asm2wasm - f64-to-int is too large ] names = names.filter((n) => !filter.includes(n)) + let successfull = 0; + let roundTripTimes = [] for (let name of names) { const spinner = startSpinner(name) const testPath = path.join(process.cwd(), 'tests', 'online', name) @@ -205,7 +228,14 @@ async function runOnlineTests(names: string[], options) { stopSpinner(spinner) cleanUpPerformance() await writeReport(name, report) + if (report.success === true) { + successfull++ + } + if (report.roundTripTime !== undefined) { + roundTripTimes.push(report.roundTripTime) + } } + writeSummary('online', names.length, successfull, roundTripTimes); } async function runOfflineTests(names: string[], options) { @@ -219,6 +249,8 @@ async function runOfflineTests(names: string[], options) { 'sqllite', ] names = names.filter((n) => !filter.includes(n)) + let successfull = 0; + let roundTripTimes = [] for (let name of names) { const spinner = startSpinner(name) const testPath = path.join(process.cwd(), 'tests', 'offline', name) @@ -229,12 +261,19 @@ async function runOfflineTests(names: string[], options) { server.close() stopSpinner(spinner) await writeReport(name, report) + if (report.success === true) { + successfull++ + } + if (report.roundTripTime !== undefined) { + roundTripTimes.push(report.roundTripTime) + } } + writeSummary('offline', names.length, successfull, roundTripTimes); } type Success = { success: true } type Failure = { success: false, reason: string } -type TestReport = { testPath: string } & (Success | Failure) +type TestReport = { testPath: string, roundTripTime?: DOMHighResTimeStamp } & (Success | Failure) async function writeReport(name: string, report: TestReport) { const totalLength = 45; if (totalLength < name.length) { @@ -246,11 +285,15 @@ async function writeReport(name: string, report: TestReport) { const testReportPath = path.join(report.testPath, 'report.txt') if (report.success === true) { await fs.writeFile(testReportPath, 'Test successfull') - process.stdout.write(`\u2713\n`) + process.stdout.write(`\u2713`) } else { - process.stdout.write(`\u2717\t\t${testReportPath}\n`) + process.stdout.write(`\u2717\t\t${testReportPath}`) await fs.writeFile(testReportPath, report.reason) } + if (report.roundTripTime !== undefined) { + process.stdout.write(`\t\tround-trip-time: ${report.roundTripTime}`) + } + process.stdout.write(`\n`) } function startServer(websitePath: string): Promise { @@ -270,6 +313,7 @@ async function testWebPage(testPath: string, options): Promise { const benchmarkPath = path.join(testPath, 'benchmark') let analysisResult: AnalysisResult try { + const p_roundTrip = createMeasure('round-trip time', { description: 'The time it takes to start the browser instance, load the webpage, record the interaction, download the data, and generate the replay', phase: 'all' }) const analyser = new Analyser('./dist/src/tracer.cjs', { extended }) analysisResult = await (await import(testJsPath)).default(analyser) const blockExtended = analyser.getExtended() @@ -284,12 +328,14 @@ async function testWebPage(testPath: string, options): Promise { // process.stdout.write(` -e not available`) const benchmark = Benchmark.fromAnalysisResult(analysisResult) await benchmark.save(benchmarkPath, options) + let m = p_roundTrip() + let roundTripTime = m.duration if (options.jsBackend != true) { - return { testPath, success: true } + return { testPath, roundTripTime, success: true } } let subBenchmarkNames = await getDirectoryNames(benchmarkPath) if (subBenchmarkNames.length === 0) { - return { testPath, success: false, reason: 'no benchmark was generated' } + return { testPath, roundTripTime, success: false, reason: 'no benchmark was generated' } } let runtimes = benchmark.instrumentBinaries() @@ -319,6 +365,7 @@ async function testWebPage(testPath: string, options): Promise { // 5. Check if original trace and replay trace match const newResult = compareResults(testPath, traceString, replayTraceString) + results.roundTripTime = roundTripTime if (newResult.success === false) { results.success = false; if (results.reason === undefined) {