From ee5d7bd61f6a7ac48d2bdc828311aa9275d08922 Mon Sep 17 00:00:00 2001 From: Luke Bermingham <1215582+lukehb@users.noreply.github.com> Date: Wed, 12 Feb 2025 17:56:04 +1300 Subject: [PATCH] Made it clearer whether video timing or abs-capture-time is used --- .../tests/peerconnection.spec.ts | 49 ++++++++++++++++--- .../LatencyCalculator.ts | 28 +++++++---- .../PeerConnectionController.ts | 2 +- Frontend/ui-library/src/UI/StatsPanel.ts | 8 +-- 4 files changed, 66 insertions(+), 21 deletions(-) diff --git a/Extras/FrontendTests/tests/peerconnection.spec.ts b/Extras/FrontendTests/tests/peerconnection.spec.ts index 41459fe7..bb62d0bb 100644 --- a/Extras/FrontendTests/tests/peerconnection.spec.ts +++ b/Extras/FrontendTests/tests/peerconnection.spec.ts @@ -96,7 +96,7 @@ test('Test abs-capture-time header extension found in PSInfra frontend', { expect(answer.sdp).toContain("abs-capture-time"); }); -test('Test latency calculation', { +test('Test video-timing header extension found in PSInfra frontend', { tag: ['@capture-time'], }, async ({ page, streamerPage, streamerId, browserName }) => { @@ -109,18 +109,52 @@ test('Test latency calculation', { await page.waitForLoadState("load"); - // Enable the flag for the capture extension - await page.evaluate(() => { - window.pixelStreaming.config.setFlagEnabled("EnableCaptureTimeExt", true); + // Wait for the sdp answer + let getSdpAnswer = new Promise(async (resolve) => { + + // Expose the resolve function to the browser context + await page.exposeFunction('resolveFromSdpAnswerPromise', resolve); + + page.evaluate(() => { + window.pixelStreaming.addEventListener("webRtcSdpAnswer", (e: WebRtcSdpAnswerEvent) => { + resolveFromSdpAnswerPromise(e.data.sdp); + }); + }); + }); + await helpers.startAndWaitForVideo(page); + const answer: RTCSessionDescriptionInit = await getSdpAnswer; + + expect(answer).toBeDefined(); + expect(answer.sdp).toBeDefined(); + + // If this string is found in the sdp we can say we have turned on the capture time header extension on the streamer + expect(answer.sdp).toContain("video-timing"); +}); + +test('Test latency calculation with video timing', { + tag: ['@video-timing'], +}, async ({ page, streamerPage, streamerId, browserName }) => { + + if(browserName !== 'chromium') { + // Chrome based browsers are the only ones that support. + test.skip(); + } + + await page.goto(`/?StreamerId=${streamerId}`); + + await page.waitForLoadState("load"); + await helpers.startAndWaitForVideo(page); // Wait for the latency info event to be fired let latencyInfo: LatencyInfo = await page.evaluate(() => { return new Promise((resolve) => { window.pixelStreaming.addEventListener("latencyCalculated", (e: LatencyCalculatedEvent) => { - if(e.data.latencyInfo && e.data.latencyInfo.senderLatencyMs && e.data.latencyInfo.rttMs) { + if(e.data.latencyInfo && e.data.latencyInfo.frameTiming && + e.data.latencyInfo.frameTiming.captureToSendLatencyMs && + e.data.latencyInfo.rttMs) { resolve(e.data.latencyInfo); } }); @@ -128,7 +162,8 @@ test('Test latency calculation', { }); expect(latencyInfo).toBeDefined(); - expect(latencyInfo.senderLatencyMs).toBeDefined(); + expect(latencyInfo.frameTiming).toBeDefined(); + expect(latencyInfo.frameTiming?.captureToSendLatencyMs).toBeDefined(); expect(latencyInfo.averageJitterBufferDelayMs).toBeDefined(); expect(latencyInfo.averageProcessingDelayMs).toBeDefined(); expect(latencyInfo.rttMs).toBeDefined(); @@ -136,7 +171,7 @@ test('Test latency calculation', { expect(latencyInfo.averageDecodeLatencyMs).toBeDefined(); // Sender side latency should be less than 500ms in pure CPU test - expect(latencyInfo.senderLatencyMs).toBeLessThanOrEqual(500) + expect(latencyInfo.frameTiming?.captureToSendLatencyMs).toBeLessThanOrEqual(500) // Expect jitter buffer/processing delay to be no greater than 500ms on local link expect(latencyInfo.averageJitterBufferDelayMs).toBeLessThanOrEqual(500); diff --git a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts index c67291eb..c06f4be8 100644 --- a/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts +++ b/Frontend/library/src/PeerConnectionController/LatencyCalculator.ts @@ -145,20 +145,23 @@ export class LatencyCalculator { ); } - // If we could not calculate latency because `senderLatencyMs` was missing because of no SenderReport yet - // We can try to substitute frame timing capture to send latency instead. + // Calculate E2E latency using video-timing capture to send time + one way network latency + receiver-side latency if ( - latencyInfo.senderLatencyMs === undefined && latencyInfo.frameTiming !== undefined && - latencyInfo.frameTiming.captureToSendLatencyMs !== undefined + latencyInfo.frameTiming.captureToSendLatencyMs !== undefined && + latencyInfo.averageProcessingDelayMs !== undefined && + latencyInfo.rttMs !== undefined ) { - latencyInfo.senderLatencyMs = latencyInfo.frameTiming.captureToSendLatencyMs; + latencyInfo.averageE2ELatency = + latencyInfo.frameTiming.captureToSendLatencyMs + + latencyInfo.rttMs * 0.5 + + latencyInfo.averageProcessingDelayMs; } - // Calculate E2E latency as sender-side latency + one way network latency + receiver-side latency + // Calculate E2E latency as abs-capture-time capture to send latency + one way network latency + receiver-side latency if ( - latencyInfo.averageProcessingDelayMs !== undefined && latencyInfo.senderLatencyMs != undefined && + latencyInfo.averageProcessingDelayMs !== undefined && latencyInfo.rttMs !== undefined ) { latencyInfo.averageE2ELatency = @@ -353,12 +356,19 @@ export class LatencyCalculator { */ export class LatencyInfo { /** - * The time taken from sender frame capture to receiver frame receipt. + * The time taken from the moment a frame is done capturing to the moment it is sent over the network. * Note: This can only be calculated if both offer and answer contain the - * the RTP header extension for `abs-capture-time`. + * the RTP header extension for `video-timing` (Chrome only for now) */ public senderLatencyMs: number | undefined = undefined; + /** + * The time taken from the moment a frame is done capturing to the moment it is sent over the network. + * Note: This can only be calculated if both offer and answer contain the + * the RTP header extension for `abs-capture-time` (Chrome only for now) + */ + public senderLatencyAbsCaptureTimeMs: number | undefined = undefined; + /* The round trip time (milliseconds) between each sender->receiver->sender */ public rttMs: number | undefined = undefined; diff --git a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts index cb2159b4..0942f86c 100644 --- a/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts +++ b/Frontend/library/src/PeerConnectionController/PeerConnectionController.ts @@ -272,7 +272,7 @@ export class PeerConnectionController { } isFirefox(): boolean { - return typeof (window as any).InstallTrigger !== 'undefined'; + return navigator.userAgent.indexOf('Firefox') > 0; } /** diff --git a/Frontend/ui-library/src/UI/StatsPanel.ts b/Frontend/ui-library/src/UI/StatsPanel.ts index 8521caa6..e1f34c43 100644 --- a/Frontend/ui-library/src/UI/StatsPanel.ts +++ b/Frontend/ui-library/src/UI/StatsPanel.ts @@ -414,8 +414,8 @@ export class StatsPanel { // Sender latency calculated using timing stats if (latencyInfo.frameTiming.captureToSendLatencyMs !== undefined) { this.addOrUpdateLatencyStat( - 'CaptureToSend', - 'Capture to send latency (ms)', + 'VideoTimingCaptureToSend', + 'Post-capture to send latency (ms)', Math.ceil(latencyInfo.frameTiming.captureToSendLatencyMs).toString() ); } @@ -423,8 +423,8 @@ export class StatsPanel { if (latencyInfo.senderLatencyMs !== undefined) { this.addOrUpdateLatencyStat( - 'SenderSideLatency', - 'Sender latency (ms)', + 'AbsCaptureTimeToSendLatency', + 'Post-capture (abs-ct) to send latency (ms)', Math.ceil(latencyInfo.senderLatencyMs).toString() ); }