diff --git a/package-lock.json b/package-lock.json index 9af174f0846b..4fd7d999268f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -160,6 +160,9 @@ "@octokit/core": "4.0.4", "@octokit/plugin-paginate-rest": "3.1.0", "@octokit/plugin-throttling": "4.1.0", + "@perf-profiler/profiler": "^0.10.9", + "@perf-profiler/reporter": "^0.8.1", + "@perf-profiler/types": "^0.8.0", "@react-native-community/eslint-config": "3.2.0", "@react-native/babel-preset": "^0.73.21", "@react-native/metro-config": "^0.73.5", @@ -7668,6 +7671,116 @@ "react-native": ">=0.70.0 <1.0.x" } }, + "node_modules/@perf-profiler/android": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@perf-profiler/android/-/android-0.12.0.tgz", + "integrity": "sha512-wLI3D63drtqw3p7aKci+LCtN/ZipLJQvcw8cfmhwxqqRxTraFa8lDz5CNvNsqtCI7Zl0N9VRtnDMOj4e1W1yMQ==", + "dev": true, + "dependencies": { + "@perf-profiler/logger": "^0.3.3", + "@perf-profiler/profiler": "^0.10.9", + "@perf-profiler/types": "^0.8.0", + "commander": "^12.0.0", + "lodash": "^4.17.21" + }, + "bin": { + "perf-profiler-commands": "dist/src/commands.js" + } + }, + "node_modules/@perf-profiler/android/node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@perf-profiler/ios": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@perf-profiler/ios/-/ios-0.3.1.tgz", + "integrity": "sha512-zRAgxLuCHzo47SYynljf+Aplh2K4DMwJ4dqIU30P8uPHiV5yHjE83eH+sTD6I7jUnUvZ8qAO1dhvp6ATJEpP/Q==", + "dev": true, + "dependencies": { + "@perf-profiler/ios-instruments": "^0.3.1", + "@perf-profiler/logger": "^0.3.3", + "@perf-profiler/types": "^0.8.0" + } + }, + "node_modules/@perf-profiler/ios-instruments": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@perf-profiler/ios-instruments/-/ios-instruments-0.3.1.tgz", + "integrity": "sha512-6ZiN9QTmIT8N37SslzjYNk+4+FX0X4IVuM/KiJF/DVgs056CT3MRDF8FFKF17BHsDJBi2a25QkegU8+AQdh+Qg==", + "dev": true, + "dependencies": { + "@perf-profiler/logger": "^0.3.3", + "@perf-profiler/profiler": "^0.10.9", + "@perf-profiler/types": "^0.8.0", + "commander": "^12.0.0", + "fast-xml-parser": "^4.2.7" + }, + "bin": { + "flashlight-ios-poc": "dist/launchIOS.js" + } + }, + "node_modules/@perf-profiler/ios-instruments/node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@perf-profiler/logger": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@perf-profiler/logger/-/logger-0.3.3.tgz", + "integrity": "sha512-iAJJ5gWhJ3zEpdMT7M2+HX0Q0UjSuCOZiEs5g8UKKPFYQjmPWwC6otHoZz6ZzRRddjiA065iD2PTytVFkpFTeQ==", + "dev": true, + "dependencies": { + "kleur": "^4.1.5", + "luxon": "^3.4.4" + }, + "bin": { + "perf-profiler-logger": "dist/bin.js" + } + }, + "node_modules/@perf-profiler/logger/node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/@perf-profiler/profiler": { + "version": "0.10.9", + "resolved": "https://registry.npmjs.org/@perf-profiler/profiler/-/profiler-0.10.9.tgz", + "integrity": "sha512-jhkFyqsrmkI9gCYmK7+R1e+vjWGw2a2YnqruRAUk71saOkLLvRSWKnT0MiGMqzi0aQj//ojeW9viDJgxQB86zg==", + "dev": true, + "dependencies": { + "@perf-profiler/android": "^0.12.0", + "@perf-profiler/ios": "^0.3.1", + "@perf-profiler/types": "^0.8.0" + } + }, + "node_modules/@perf-profiler/reporter": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@perf-profiler/reporter/-/reporter-0.8.1.tgz", + "integrity": "sha512-lZp17uMMLAV4nuDO0JbajbPCyOoD4/ugnZVxsOEEueRo8mxB26TS3R7ANtMZYjHrpQbJry0CgfTIPxflBgtq4A==", + "dev": true, + "dependencies": { + "@perf-profiler/types": "^0.8.0", + "lodash": "^4.17.21" + } + }, + "node_modules/@perf-profiler/types": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@perf-profiler/types/-/types-0.8.0.tgz", + "integrity": "sha512-TFiktv00SzLjjPp1hFYYjT9O36iGIUaF6yPLd7x/UT4CuLd0YYDUj+gvX0fbXtVtV7141tTvWbXFL5HiXGx0kw==", + "dev": true + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "license": "MIT", @@ -27660,6 +27773,15 @@ "node": ">=10" } }, + "node_modules/luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.9", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.9.tgz", diff --git a/package.json b/package.json index 3d679bd79557..73be247cf568 100644 --- a/package.json +++ b/package.json @@ -213,6 +213,9 @@ "@octokit/core": "4.0.4", "@octokit/plugin-paginate-rest": "3.1.0", "@octokit/plugin-throttling": "4.1.0", + "@perf-profiler/profiler": "^0.10.9", + "@perf-profiler/reporter": "^0.8.1", + "@perf-profiler/types": "^0.8.0", "@react-native-community/eslint-config": "3.2.0", "@react-native/babel-preset": "^0.73.21", "@react-native/metro-config": "^0.73.5", diff --git a/patches/@perf-profiler+android+0.12.0.patch b/patches/@perf-profiler+android+0.12.0.patch new file mode 100644 index 000000000000..f6ecbce9b481 --- /dev/null +++ b/patches/@perf-profiler+android+0.12.0.patch @@ -0,0 +1,54 @@ +diff --git a/node_modules/@perf-profiler/android/dist/src/commands.js b/node_modules/@perf-profiler/android/dist/src/commands.js +old mode 100755 +new mode 100644 +diff --git a/node_modules/@perf-profiler/android/dist/src/commands/platforms/UnixProfiler.js b/node_modules/@perf-profiler/android/dist/src/commands/platforms/UnixProfiler.js +index 77b9ee0..59aeed9 100644 +--- a/node_modules/@perf-profiler/android/dist/src/commands/platforms/UnixProfiler.js ++++ b/node_modules/@perf-profiler/android/dist/src/commands/platforms/UnixProfiler.js +@@ -134,7 +134,20 @@ class UnixProfiler { + } + const subProcessesStats = (0, getCpuStatsByProcess_1.processOutput)(cpu, pid); + const ram = (0, pollRamUsage_1.processOutput)(ramStr, this.getRAMPageSize()); +- const { frameTimes, interval: atraceInterval } = frameTimeParser.getFrameTimes(atrace, pid); ++ ++ let output; ++ try { ++ output = frameTimeParser.getFrameTimes(atrace, pid); ++ } catch (e) { ++ console.error(e); ++ } ++ ++ if (!output) { ++ return; ++ } ++ ++ const { frameTimes, interval: atraceInterval } = output; ++ + if (!initialTime) { + initialTime = timestamp; + } +diff --git a/node_modules/@perf-profiler/android/src/commands/platforms/UnixProfiler.ts b/node_modules/@perf-profiler/android/src/commands/platforms/UnixProfiler.ts +index d6983c1..ccacf09 100644 +--- a/node_modules/@perf-profiler/android/src/commands/platforms/UnixProfiler.ts ++++ b/node_modules/@perf-profiler/android/src/commands/platforms/UnixProfiler.ts +@@ -136,7 +136,19 @@ export abstract class UnixProfiler implements Profiler { + const subProcessesStats = processOutput(cpu, pid); + + const ram = processRamOutput(ramStr, this.getRAMPageSize()); +- const { frameTimes, interval: atraceInterval } = frameTimeParser.getFrameTimes(atrace, pid); ++ ++ let output; ++ try { ++ output = frameTimeParser.getFrameTimes(atrace, pid); ++ } catch (e) { ++ console.error(e); ++ } ++ ++ if (!output) { ++ return; ++ } ++ ++ const { frameTimes, interval: atraceInterval } = output; + + if (!initialTime) { + initialTime = timestamp; diff --git a/patches/@perf-profiler+reporter+0.8.1.patch b/patches/@perf-profiler+reporter+0.8.1.patch new file mode 100644 index 000000000000..2c918b4049c2 --- /dev/null +++ b/patches/@perf-profiler+reporter+0.8.1.patch @@ -0,0 +1,25 @@ +diff --git a/node_modules/@perf-profiler/reporter/dist/src/index.d.ts b/node_modules/@perf-profiler/reporter/dist/src/index.d.ts +index 2f84d84..14ae688 100644 +--- a/node_modules/@perf-profiler/reporter/dist/src/index.d.ts ++++ b/node_modules/@perf-profiler/reporter/dist/src/index.d.ts +@@ -4,4 +4,6 @@ export * from "./reporting/Report"; + export * from "./utils/sanitizeProcessName"; + export * from "./utils/round"; + export * from "./reporting/cpu"; ++export * from "./reporting/ram"; ++export * from "./reporting/fps"; + export { canComputeHighCpuUsage } from "./reporting/highCpu"; +diff --git a/node_modules/@perf-profiler/reporter/dist/src/index.js b/node_modules/@perf-profiler/reporter/dist/src/index.js +index 4b50e3a..780963a 100644 +--- a/node_modules/@perf-profiler/reporter/dist/src/index.js ++++ b/node_modules/@perf-profiler/reporter/dist/src/index.js +@@ -21,6 +21,8 @@ __exportStar(require("./reporting/Report"), exports); + __exportStar(require("./utils/sanitizeProcessName"), exports); + __exportStar(require("./utils/round"), exports); + __exportStar(require("./reporting/cpu"), exports); ++__exportStar(require("./reporting/fps"), exports); ++__exportStar(require("./reporting/ram"), exports); + var highCpu_1 = require("./reporting/highCpu"); + Object.defineProperty(exports, "canComputeHighCpuUsage", { enumerable: true, get: function () { return highCpu_1.canComputeHighCpuUsage; } }); + //# sourceMappingURL=index.js.map +\ No newline at end of file diff --git a/tests/e2e/config.dev.ts b/tests/e2e/config.dev.ts index cdd7bce756c8..a400cbf4787e 100644 --- a/tests/e2e/config.dev.ts +++ b/tests/e2e/config.dev.ts @@ -6,6 +6,8 @@ const appPath = './android/app/build/outputs/apk/development/debug/app-developme const config: Config = { MAIN_APP_PACKAGE: packageName, DELTA_APP_PACKAGE: packageName, + BRANCH_MAIN: 'main', + BRANCH_DELTA: 'main', MAIN_APP_PATH: appPath, DELTA_APP_PATH: appPath, RUNS: 8, diff --git a/tests/e2e/config.ts b/tests/e2e/config.ts index 6eb6bb839ae2..8963e07c31c8 100644 --- a/tests/e2e/config.ts +++ b/tests/e2e/config.ts @@ -26,6 +26,9 @@ export default { MAIN_APP_PATH: './app-e2eRelease.apk', DELTA_APP_PATH: './app-e2edeltaRelease.apk', + BRANCH_MAIN: 'main', + BRANCH_DELTA: 'delta', + ENTRY_FILE: 'src/libs/E2E/reactNativeLaunchingTest.ts', // The path to the activity within the app that we want to launch. diff --git a/tests/e2e/server/index.ts b/tests/e2e/server/index.ts index 68367aa29c2a..cd1bbeca2ba9 100644 --- a/tests/e2e/server/index.ts +++ b/tests/e2e/server/index.ts @@ -20,15 +20,17 @@ type TestDoneListener = () => void; type TestResultListener = (testResult: TestResult) => void; -type AddListener = (listener: TListener) => void; +type AddListener = (listener: TListener) => () => void; type ServerInstance = { setTestConfig: (testConfig: TestConfig) => void; + getTestConfig: () => TestConfig; addTestStartedListener: AddListener; addTestResultListener: AddListener; addTestDoneListener: AddListener; forceTestCompletion: () => void; setReadyToAcceptTestResults: (isReady: boolean) => void; + isReadyToAcceptTestResults: boolean; start: () => Promise; stop: () => Promise; }; @@ -115,6 +117,13 @@ const createServerInstance = (): ServerInstance => { const setTestConfig = (testConfig: TestConfig) => { activeTestConfig = testConfig; }; + const getTestConfig = (): TestConfig => { + if (!activeTestConfig) { + throw new Error('No test config set'); + } + + return activeTestConfig; + }; const server = createServer((req, res): ServerResponse | void => { res.statusCode = 200; @@ -212,7 +221,11 @@ const createServerInstance = (): ServerInstance => { return { setReadyToAcceptTestResults, + get isReadyToAcceptTestResults() { + return isReadyToAcceptTestResults; + }, setTestConfig, + getTestConfig, addTestStartedListener, addTestResultListener, addTestDoneListener, diff --git a/tests/e2e/testRunner.ts b/tests/e2e/testRunner.ts index a8295f6ddf5c..297cbbfcce57 100644 --- a/tests/e2e/testRunner.ts +++ b/tests/e2e/testRunner.ts @@ -16,6 +16,7 @@ /* eslint-disable @lwc/lwc/no-async-await,no-restricted-syntax,no-await-in-loop */ import {execSync} from 'child_process'; import fs from 'fs'; +import type {TestResult} from '@libs/E2E/client'; import type {TestConfig} from '@libs/E2E/types'; import compare from './compare/compare'; import defaultConfig from './config'; @@ -25,6 +26,7 @@ import installApp from './utils/installApp'; import killApp from './utils/killApp'; import launchApp from './utils/launchApp'; import * as Logger from './utils/logger'; +import * as MeasureUtils from './utils/measure'; import sleep from './utils/sleep'; import withFailTimeout from './utils/withFailTimeout'; @@ -91,18 +93,7 @@ const runTests = async (): Promise => { // Create a dict in which we will store the run durations for all tests const results: Record = {}; - // Collect results while tests are being executed - server.addTestResultListener((testResult) => { - const {isCritical = true} = testResult; - - if (testResult?.error != null && isCritical) { - throw new Error(`Test '${testResult.name}' failed with error: ${testResult.error}`); - } - if (testResult?.error != null && !isCritical) { - // force test completion, since we don't want to have timeout error for non being execute test - server.forceTestCompletion(); - Logger.warn(`Test '${testResult.name}' failed with error: ${testResult.error}`); - } + const attachTestResult = (testResult: TestResult) => { let result = 0; if (testResult?.duration !== undefined) { @@ -124,10 +115,26 @@ const runTests = async (): Promise => { if (testResult?.branch && testResult?.name) { results[testResult.branch][testResult.name] = (results[testResult.branch][testResult.name] ?? []).concat(result); } + }; + + // Collect results while tests are being executed + server.addTestResultListener((testResult) => { + const {isCritical = true} = testResult; + + if (testResult?.error != null && isCritical) { + throw new Error(`Test '${testResult.name}' failed with error: ${testResult.error}`); + } + if (testResult?.error != null && !isCritical) { + // force test completion, since we don't want to have timeout error for non being execute test + server.forceTestCompletion(); + Logger.warn(`Test '${testResult.name}' failed with error: ${testResult.error}`); + } + + attachTestResult(testResult); }); // Function to run a single test iteration - async function runTestIteration(appPackage: string, iterationText: string, launchArgs: Record = {}): Promise { + async function runTestIteration(appPackage: string, iterationText: string, branch: string, launchArgs: Record = {}): Promise { Logger.info(iterationText); // Making sure the app is really killed (e.g. if a prior test run crashed) @@ -137,10 +144,43 @@ const runTests = async (): Promise => { Logger.log('Launching', appPackage); await launchApp('android', appPackage, config.ACTIVITY_PATH, launchArgs); + MeasureUtils.start(appPackage); await withFailTimeout( new Promise((resolve) => { - server.addTestDoneListener(() => { + const removeListener = server.addTestDoneListener(() => { Logger.success(iterationText); + + const metrics = MeasureUtils.stop(); + const test = server.getTestConfig(); + + if (server.isReadyToAcceptTestResults) { + attachTestResult({ + name: `${test.name} (CPU)`, + branch, + duration: metrics.cpu, + }); + attachTestResult({ + name: `${test.name} (FPS)`, + branch, + duration: metrics.fps, + }); + attachTestResult({ + name: `${test.name} (RAM)`, + branch, + duration: metrics.ram, + }); + attachTestResult({ + name: `${test.name} (CPU/JS)`, + branch, + duration: metrics.jsThread, + }); + attachTestResult({ + name: `${test.name} (CPU/UI)`, + branch, + duration: metrics.uiThread, + }); + } + removeListener(); resolve(); }); }), @@ -188,10 +228,10 @@ const runTests = async (): Promise => { const iterations = 2; for (let i = 0; i < iterations; i++) { // Warmup the main app: - await runTestIteration(config.MAIN_APP_PACKAGE, `[MAIN] ${warmupText}. Iteration ${i + 1}/${iterations}`); + await runTestIteration(config.MAIN_APP_PACKAGE, `[MAIN] ${warmupText}. Iteration ${i + 1}/${iterations}`, config.BRANCH_MAIN); // Warmup the delta app: - await runTestIteration(config.DELTA_APP_PACKAGE, `[DELTA] ${warmupText}. Iteration ${i + 1}/${iterations}`); + await runTestIteration(config.DELTA_APP_PACKAGE, `[DELTA] ${warmupText}. Iteration ${i + 1}/${iterations}`, config.BRANCH_DELTA); } server.setReadyToAcceptTestResults(true); @@ -226,10 +266,10 @@ const runTests = async (): Promise => { const deltaIterationText = `[DELTA] ${iterationText}`; try { // Run the test on the main app: - await runTestIteration(config.MAIN_APP_PACKAGE, mainIterationText, launchArgs); + await runTestIteration(config.MAIN_APP_PACKAGE, mainIterationText, config.BRANCH_MAIN, launchArgs); // Run the test on the delta app: - await runTestIteration(config.DELTA_APP_PACKAGE, deltaIterationText, launchArgs); + await runTestIteration(config.DELTA_APP_PACKAGE, deltaIterationText, config.BRANCH_DELTA, launchArgs); } catch (e) { onError(e as Error); } diff --git a/tests/e2e/utils/measure.ts b/tests/e2e/utils/measure.ts new file mode 100644 index 000000000000..96ac1bb4541e --- /dev/null +++ b/tests/e2e/utils/measure.ts @@ -0,0 +1,46 @@ +import {profiler} from '@perf-profiler/profiler'; +import {getAverageCpuUsage, getAverageCpuUsagePerProcess, getAverageFPSUsage, getAverageRAMUsage} from '@perf-profiler/reporter'; +import {ThreadNames} from '@perf-profiler/types'; +import type {Measure} from '@perf-profiler/types'; + +let measures: Measure[] = []; +const POLLING_STOPPED = { + stop: (): void => { + throw new Error('Cannot stop polling on a stopped profiler'); + }, +}; +let polling = POLLING_STOPPED; + +const start = (bundleId: string) => { + // clear our measurements results + measures = []; + + polling = profiler.pollPerformanceMeasures(bundleId, { + onMeasure: (measure: Measure) => { + measures.push(measure); + }, + }); +}; + +const stop = () => { + polling.stop(); + polling = POLLING_STOPPED; + + const average = getAverageCpuUsagePerProcess(measures); + const uiThread = average.find(({processName}) => processName === ThreadNames.ANDROID.UI)?.cpuUsage; + // most likely this line needs to be updated when we migrate to RN 0.74 with bridgeless mode + const jsThread = average.find(({processName}) => processName === ThreadNames.RN.JS_ANDROID)?.cpuUsage; + const cpu = getAverageCpuUsage(measures); + const fps = getAverageFPSUsage(measures); + const ram = getAverageRAMUsage(measures); + + return { + uiThread, + jsThread, + cpu, + fps, + ram, + }; +}; + +export {start, stop};