From bee3e7f44d86af55858f0878906b797456b17861 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 17 Sep 2024 01:08:05 +0200 Subject: [PATCH] add task-graph-time audit --- code-pushup.config.ts | 8 ++- docs/motivation.md | 2 +- .../audit/project-graph.audit.ts | 4 +- .../nx-performance/audit/task-graph.audit.ts | 69 +++++++++++++++++++ .../nx-performance/nx-performance.plugin.ts | 26 ++++++- 5 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 tooling/measures/nx-performance/audit/task-graph.audit.ts diff --git a/code-pushup.config.ts b/code-pushup.config.ts index 6cfff029..aa5d8e09 100644 --- a/code-pushup.config.ts +++ b/code-pushup.config.ts @@ -5,10 +5,12 @@ import nxPerformancePlugin, { } from './tooling/measures/nx-performance/nx-performance.plugin'; const onlyAudits: OnlyAudit[] = [ - 'project-graph-performance', + 'project-graph-time', 'task-time', 'cache-size', + 'task-graph-time', ]; +const taskGraphTasks = ['cli-e2e:install-env']; const taskTimeTasks = [ 'cli-e2e:e2e', 'core-e2e:e2e', @@ -27,7 +29,10 @@ export default { plugins: [ nxPerformancePlugin({ taskTimeTasks, + taskGraphTasks, + maxTaskTime: 60 * 1000 * 1.5, cacheSizeTasks, + maxCacheSize: 6000000, onlyAudits, }), ], @@ -39,6 +44,7 @@ export default { ...nxPerformanceCategoryRefs({ taskTimeTasks, cacheSizeTasks, + taskGraphTasks, onlyAudits, }), ], diff --git a/docs/motivation.md b/docs/motivation.md index 60ee2a18..e4f99954 100644 --- a/docs/motivation.md +++ b/docs/motivation.md @@ -75,7 +75,7 @@ export async function teardown() { Now you could run `nx run my-lib-e2e:e2e` which would start the server publish and install, executes the tests and runs the cleanup logic. Viola, you have a working e2e setup for your package. 🎉 -But wait! There are MANY caveats with this setup. We mentioned them already in the beginning, now let's discuss them one by one. +**But wait!** There are MANY caveats with this setup. We mentioned them already in the beginning, now let's discuss them one by one. ## Problems diff --git a/tooling/measures/nx-performance/audit/project-graph.audit.ts b/tooling/measures/nx-performance/audit/project-graph.audit.ts index 1552c14a..61ec74cf 100644 --- a/tooling/measures/nx-performance/audit/project-graph.audit.ts +++ b/tooling/measures/nx-performance/audit/project-graph.audit.ts @@ -6,7 +6,7 @@ import { executeProcess, slugify } from '@code-pushup/utils'; export const DEFAULT_MAX_PROJECT_GRAPH_TIME = 300; -export const PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG = 'project-graph-performance'; +export const PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG = 'project-graph-time'; export const PROJECT_GRAPH_PERFORMANCE_AUDIT = { slug: PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG, title: 'Project graph performance', @@ -25,7 +25,7 @@ export async function projectGraphAudit( const { duration } = await projectGraphTiming(); return { - slug: 'project-graph-performance', + slug: PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG, score: scoreProjectGraphDuration(duration, maxProjectGraphTime), value: duration, displayValue: `${duration.toFixed(2)} ms`, diff --git a/tooling/measures/nx-performance/audit/task-graph.audit.ts b/tooling/measures/nx-performance/audit/task-graph.audit.ts new file mode 100644 index 00000000..a12c18b0 --- /dev/null +++ b/tooling/measures/nx-performance/audit/task-graph.audit.ts @@ -0,0 +1,69 @@ +import { Audit, AuditOutputs } from '@code-pushup/models'; +import { execFile } from 'node:child_process'; +import { slugify } from '@code-pushup/utils'; + +export const DEFAULT_MAX_TASK_GRAPH_TIME = 300; +export const TASK_GRAPH_TIME_AUDIT_POSTFIX = 'task-graph-time'; + +export function getTaskGraphTimeAuditSlug(task: string): string { + return `${slugify(task)}-${TASK_GRAPH_TIME_AUDIT_POSTFIX}`; +} + +export const getTaskGraphTimeAudits = (tasks: string[]): Audit[] => { + return tasks.map((task) => ({ + slug: getTaskGraphTimeAuditSlug(task), // Unique slug for each task + title: 'Task graph performance', + description: 'An audit to check performance of the Nx task graph', + })); +}; + +export type TaskGraphAuditOptions = { + taskGraphTasks: string[]; + maxTaskGraphTime?: number; +}; + +export async function taskGraphAudits( + options?: TaskGraphAuditOptions +): Promise { + const { maxTaskGraphTime = DEFAULT_MAX_TASK_GRAPH_TIME, taskGraphTasks } = + options ?? {}; + const results = await taskGraphTiming(taskGraphTasks); + + return results.map(({ duration, task }) => ({ + slug: getTaskGraphTimeAuditSlug(task), + score: scoreTaskGraphDuration(duration, maxTaskGraphTime), + value: duration, + displayValue: `${duration.toFixed(2)} ms`, + details: {}, + })); +} + +export function scoreTaskGraphDuration( + duration: number, + maxDuration: number +): number { + // Ensure duration is capped at maxDuration for the scoring + if (duration >= maxDuration) return 0; + + // A simple linear score where a lower duration gives a higher score. + // When duration == 0, score is 1 (perfect). When duration == maxDuration, score is 0 (poor). + return 1 - duration / maxDuration; +} + +export async function taskGraphTiming( + tasks: string[] +): Promise<{ duration: number; task: string }[]> { + const results: { duration: number; task: string }[] = []; + for (const task of tasks) { + const start = performance.now(); + execFile( + `NX_DAEMON=true NX_CACHE_PROJECT_GRAPH=false NX_ISOLATE_PLUGINS=true npx nx run-many -t ${task} --graph tmp/nx-performance/task-graph/${task}.graph.json` + ); + const execFileDuration = Number((performance.now() - start).toFixed(3)); + results.push({ + task, + duration: Number(execFileDuration.toFixed(3)), + }); + } + return results; +} diff --git a/tooling/measures/nx-performance/nx-performance.plugin.ts b/tooling/measures/nx-performance/nx-performance.plugin.ts index c0d4c014..100ac4f1 100644 --- a/tooling/measures/nx-performance/nx-performance.plugin.ts +++ b/tooling/measures/nx-performance/nx-performance.plugin.ts @@ -23,14 +23,22 @@ import { cacheSizeAudits, getCacheSizeAudits, } from './audit/cache-size.audit'; +import { + getTaskGraphTimeAudits, + TASK_GRAPH_TIME_AUDIT_POSTFIX, + TaskGraphAuditOptions, + taskGraphAudits, +} from './audit/task-graph.audit'; export const nxPerformanceAudits = ({ taskTimeTasks, cacheSizeTasks, + taskGraphTasks, }: NxPerfPluginConfig) => [ PROJECT_GRAPH_PERFORMANCE_AUDIT, ...(taskTimeTasks ? getTaskTimeAudits(taskTimeTasks) : []), ...(cacheSizeTasks ? getCacheSizeAudits(cacheSizeTasks) : []), + ...(taskGraphTasks ? getTaskGraphTimeAudits(taskGraphTasks) : []), ]; export const nxPerformanceCategoryRefs = ( @@ -51,12 +59,14 @@ export const nxPerformanceCategoryRefs = ( export type OnlyAudit = | typeof CACHE_SIZE_AUDIT_POSTFIX | typeof PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG - | typeof TASK_TIME_AUDIT_POSTFIX; + | typeof TASK_TIME_AUDIT_POSTFIX + | typeof TASK_GRAPH_TIME_AUDIT_POSTFIX; export type NxPerfPluginConfig = { onlyAudits?: OnlyAudit[]; } & ProjectGraphAuditOptions & ProjectTaskAuditOptions & - CacheSizeAuditOptions; + CacheSizeAuditOptions & + TaskGraphAuditOptions; export function nxPerformancePlugin( options?: NxPerfPluginConfig @@ -90,6 +100,12 @@ export function filterOnlyAudits( ) { return true; } + if ( + onlyAuditsSet.has(TASK_GRAPH_TIME_AUDIT_POSTFIX) && + slug.endsWith(TASK_GRAPH_TIME_AUDIT_POSTFIX) + ) { + return true; + } if ( onlyAuditsSet.has(TASK_TIME_AUDIT_POSTFIX) && slug.endsWith(TASK_TIME_AUDIT_POSTFIX) @@ -108,8 +124,11 @@ export async function runnerFunction( PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG, CACHE_SIZE_AUDIT_POSTFIX, TASK_TIME_AUDIT_POSTFIX, + TASK_GRAPH_TIME_AUDIT_POSTFIX, ], taskTimeTasks, + taskGraphTasks, + maxTaskGraphTime, maxTaskTime, maxCacheSize, cacheSizeTasks, @@ -123,6 +142,9 @@ export async function runnerFunction( ...(onlyAuditsSet.has(CACHE_SIZE_AUDIT_POSTFIX) ? await cacheSizeAudits({ maxCacheSize, cacheSizeTasks }) : []), + ...(onlyAuditsSet.has(TASK_GRAPH_TIME_AUDIT_POSTFIX) + ? await taskGraphAudits({ maxTaskGraphTime, taskGraphTasks }) + : []), ...(onlyAuditsSet.has(TASK_TIME_AUDIT_POSTFIX) ? await taskTimeAudits({ maxTaskTime, taskTimeTasks }) : []),