bench: add task-graph-time audit

code-pushup.config.ts

@@ -5,10 +5,12 @@ import nxPerformancePlugin, {
 } from './tooling/measures/nx-performance/nx-performance.plugin';
 
 const onlyAudits: OnlyAudit[] = [
-  'project-graph-performance',
+  'project-graph-time',
   'task-time',
   'cache-size',
+  'task-graph-time',
 ];
+const taskGraphTasks = ['cli-e2e:install-env'];
 const taskTimeTasks = [
   'cli-e2e:e2e',
   'core-e2e:e2e',
@@ -27,7 +29,10 @@ export default {
   plugins: [
     nxPerformancePlugin({
       taskTimeTasks,
+      taskGraphTasks,
+      maxTaskTime: 60 * 1000 * 1.5,
       cacheSizeTasks,
+      maxCacheSize: 6000000,
       onlyAudits,
     }),
   ],
@@ -39,6 +44,7 @@ export default {
         ...nxPerformanceCategoryRefs({
           taskTimeTasks,
           cacheSizeTasks,
+          taskGraphTasks,
           onlyAudits,
         }),
       ],

docs/motivation.md

@@ -75,7 +75,7 @@ export async function teardown() {
 Now you could run `nx run my-lib-e2e:e2e` which would start the server publish and install, executes the tests and runs the cleanup logic.
 Viola, you have a working e2e setup for your package. 🎉
 
-But wait! There are MANY caveats with this setup. We mentioned them already in the beginning, now let's discuss them one by one.
+**But wait!** There are MANY caveats with this setup. We mentioned them already in the beginning, now let's discuss them one by one.
 
 ## Problems
 

tooling/measures/nx-performance/audit/project-graph.audit.ts

@@ -6,7 +6,7 @@ import { executeProcess, slugify } from '@code-pushup/utils';
 
 export const DEFAULT_MAX_PROJECT_GRAPH_TIME = 300;
 
-export const PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG = 'project-graph-performance';
+export const PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG = 'project-graph-time';
 export const PROJECT_GRAPH_PERFORMANCE_AUDIT = {
   slug: PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG,
   title: 'Project graph performance',
@@ -25,7 +25,7 @@ export async function projectGraphAudit(
   const { duration } = await projectGraphTiming();
 
   return {
-    slug: 'project-graph-performance',
+    slug: PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG,
     score: scoreProjectGraphDuration(duration, maxProjectGraphTime),
     value: duration,
     displayValue: `${duration.toFixed(2)} ms`,

tooling/measures/nx-performance/audit/task-graph.audit.ts

@@ -0,0 +1,69 @@
+import { Audit, AuditOutputs } from '@code-pushup/models';
+import { execFile } from 'node:child_process';
+import { slugify } from '@code-pushup/utils';
+
+export const DEFAULT_MAX_TASK_GRAPH_TIME = 300;
+export const TASK_GRAPH_TIME_AUDIT_POSTFIX = 'task-graph-time';
+
+export function getTaskGraphTimeAuditSlug(task: string): string {
+  return `${slugify(task)}-${TASK_GRAPH_TIME_AUDIT_POSTFIX}`;
+}
+
+export const getTaskGraphTimeAudits = (tasks: string[]): Audit[] => {
+  return tasks.map((task) => ({
+    slug: getTaskGraphTimeAuditSlug(task), // Unique slug for each task
+    title: 'Task graph performance',
+    description: 'An audit to check performance of the Nx task graph',
+  }));
+};
+
+export type TaskGraphAuditOptions = {
+  taskGraphTasks: string[];
+  maxTaskGraphTime?: number;
+};
+
+export async function taskGraphAudits(
+  options?: TaskGraphAuditOptions
+): Promise<AuditOutputs> {
+  const { maxTaskGraphTime = DEFAULT_MAX_TASK_GRAPH_TIME, taskGraphTasks } =
+    options ?? {};
+  const results = await taskGraphTiming(taskGraphTasks);
+
+  return results.map(({ duration, task }) => ({
+    slug: getTaskGraphTimeAuditSlug(task),
+    score: scoreTaskGraphDuration(duration, maxTaskGraphTime),
+    value: duration,
+    displayValue: `${duration.toFixed(2)} ms`,
+    details: {},
+  }));
+}
+
+export function scoreTaskGraphDuration(
+  duration: number,
+  maxDuration: number
+): number {
+  // Ensure duration is capped at maxDuration for the scoring
+  if (duration >= maxDuration) return 0;
+
+  // A simple linear score where a lower duration gives a higher score.
+  // When duration == 0, score is 1 (perfect). When duration == maxDuration, score is 0 (poor).
+  return 1 - duration / maxDuration;
+}
+
+export async function taskGraphTiming(
+  tasks: string[]
+): Promise<{ duration: number; task: string }[]> {
+  const results: { duration: number; task: string }[] = [];
+  for (const task of tasks) {
+    const start = performance.now();
+    execFile(
+      `NX_DAEMON=true NX_CACHE_PROJECT_GRAPH=false NX_ISOLATE_PLUGINS=true npx nx run-many -t ${task} --graph tmp/nx-performance/task-graph/${task}.graph.json`
+    );
+    const execFileDuration = Number((performance.now() - start).toFixed(3));
+    results.push({
+      task,
+      duration: Number(execFileDuration.toFixed(3)),
+    });
+  }
+  return results;
+}

tooling/measures/nx-performance/nx-performance.plugin.ts

@@ -23,14 +23,22 @@ import {
   cacheSizeAudits,
   getCacheSizeAudits,
 } from './audit/cache-size.audit';
+import {
+  getTaskGraphTimeAudits,
+  TASK_GRAPH_TIME_AUDIT_POSTFIX,
+  TaskGraphAuditOptions,
+  taskGraphAudits,
+} from './audit/task-graph.audit';
 
 export const nxPerformanceAudits = ({
   taskTimeTasks,
   cacheSizeTasks,
+  taskGraphTasks,
 }: NxPerfPluginConfig) => [
   PROJECT_GRAPH_PERFORMANCE_AUDIT,
   ...(taskTimeTasks ? getTaskTimeAudits(taskTimeTasks) : []),
   ...(cacheSizeTasks ? getCacheSizeAudits(cacheSizeTasks) : []),
+  ...(taskGraphTasks ? getTaskGraphTimeAudits(taskGraphTasks) : []),
 ];
 
 export const nxPerformanceCategoryRefs = (
@@ -51,12 +59,14 @@ export const nxPerformanceCategoryRefs = (
 export type OnlyAudit =
   | typeof CACHE_SIZE_AUDIT_POSTFIX
   | typeof PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG
-  | typeof TASK_TIME_AUDIT_POSTFIX;
+  | typeof TASK_TIME_AUDIT_POSTFIX
+  | typeof TASK_GRAPH_TIME_AUDIT_POSTFIX;
 export type NxPerfPluginConfig = {
   onlyAudits?: OnlyAudit[];
 } & ProjectGraphAuditOptions &
   ProjectTaskAuditOptions &
-  CacheSizeAuditOptions;
+  CacheSizeAuditOptions &
+  TaskGraphAuditOptions;
 
 export function nxPerformancePlugin(
   options?: NxPerfPluginConfig
@@ -90,6 +100,12 @@ export function filterOnlyAudits(
     ) {
       return true;
     }
+    if (
+      onlyAuditsSet.has(TASK_GRAPH_TIME_AUDIT_POSTFIX) &&
+      slug.endsWith(TASK_GRAPH_TIME_AUDIT_POSTFIX)
+    ) {
+      return true;
+    }
     if (
       onlyAuditsSet.has(TASK_TIME_AUDIT_POSTFIX) &&
       slug.endsWith(TASK_TIME_AUDIT_POSTFIX)
@@ -108,8 +124,11 @@ export async function runnerFunction(
       PROJECT_GRAPH_PERFORMANCE_AUDIT_SLUG,
       CACHE_SIZE_AUDIT_POSTFIX,
       TASK_TIME_AUDIT_POSTFIX,
+      TASK_GRAPH_TIME_AUDIT_POSTFIX,
     ],
     taskTimeTasks,
+    taskGraphTasks,
+    maxTaskGraphTime,
     maxTaskTime,
     maxCacheSize,
     cacheSizeTasks,
@@ -123,6 +142,9 @@ export async function runnerFunction(
     ...(onlyAuditsSet.has(CACHE_SIZE_AUDIT_POSTFIX)
       ? await cacheSizeAudits({ maxCacheSize, cacheSizeTasks })
       : []),
+    ...(onlyAuditsSet.has(TASK_GRAPH_TIME_AUDIT_POSTFIX)
+      ? await taskGraphAudits({ maxTaskGraphTime, taskGraphTasks })
+      : []),
     ...(onlyAuditsSet.has(TASK_TIME_AUDIT_POSTFIX)
       ? await taskTimeAudits({ maxTaskTime, taskTimeTasks })
       : []),