Made it clearer whether video timing or abs-capture-time is used

Extras/FrontendTests/tests/peerconnection.spec.ts

@@ -96,7 +96,7 @@ test('Test abs-capture-time header extension found in PSInfra frontend', {
     expect(answer.sdp).toContain("abs-capture-time");
 });
 
-test('Test latency calculation', {
+test('Test video-timing header extension found in PSInfra frontend', {
     tag: ['@capture-time'],
 }, async ({ page, streamerPage, streamerId, browserName }) => {
 
@@ -109,34 +109,69 @@ test('Test latency calculation', {
 
     await page.waitForLoadState("load");
 
-    // Enable the flag for the capture extension
-    await page.evaluate(() => {
-        window.pixelStreaming.config.setFlagEnabled("EnableCaptureTimeExt", true);
+    // Wait for the sdp answer
+    let getSdpAnswer = new Promise<RTCSessionDescriptionInit>(async (resolve) => {
+
+        // Expose the resolve function to the browser context
+        await page.exposeFunction('resolveFromSdpAnswerPromise', resolve);
+
+        page.evaluate(() => {
+            window.pixelStreaming.addEventListener("webRtcSdpAnswer", (e: WebRtcSdpAnswerEvent) => {
+                resolveFromSdpAnswerPromise(e.data.sdp);
+            });
+        });
+
     });
 
+    await helpers.startAndWaitForVideo(page);
+    const answer: RTCSessionDescriptionInit = await getSdpAnswer;
+
+    expect(answer).toBeDefined();
+    expect(answer.sdp).toBeDefined();
+
+    // If this string is found in the sdp we can say we have turned on the capture time header extension on the streamer
+    expect(answer.sdp).toContain("video-timing");
+});
+
+test('Test latency calculation with video timing', {
+    tag: ['@video-timing'],
+}, async ({ page, streamerPage, streamerId, browserName }) => {
+
+    if(browserName !== 'chromium') {
+        // Chrome based browsers are the only ones that support.
+        test.skip();
+    }
+
+    await page.goto(`/?StreamerId=${streamerId}`);
+
+    await page.waitForLoadState("load");
+
     await helpers.startAndWaitForVideo(page);
 
     // Wait for the latency info event to be fired
     let latencyInfo: LatencyInfo = await page.evaluate(() => {
         return new Promise((resolve) => {
             window.pixelStreaming.addEventListener("latencyCalculated", (e: LatencyCalculatedEvent) => {
-                if(e.data.latencyInfo && e.data.latencyInfo.senderLatencyMs && e.data.latencyInfo.rttMs) {
+                if(e.data.latencyInfo && e.data.latencyInfo.frameTiming && 
+                    e.data.latencyInfo.frameTiming.captureToSendLatencyMs && 
+                    e.data.latencyInfo.rttMs) {
                     resolve(e.data.latencyInfo);
                 }
             });
         });
     });
 
     expect(latencyInfo).toBeDefined();
-    expect(latencyInfo.senderLatencyMs).toBeDefined();
+    expect(latencyInfo.frameTiming).toBeDefined();
+    expect(latencyInfo.frameTiming?.captureToSendLatencyMs).toBeDefined();
     expect(latencyInfo.averageJitterBufferDelayMs).toBeDefined();
     expect(latencyInfo.averageProcessingDelayMs).toBeDefined();
     expect(latencyInfo.rttMs).toBeDefined();
     expect(latencyInfo.averageAssemblyDelayMs).toBeDefined();
     expect(latencyInfo.averageDecodeLatencyMs).toBeDefined();
 
     // Sender side latency should be less than 500ms in pure CPU test
-    expect(latencyInfo.senderLatencyMs).toBeLessThanOrEqual(500)
+    expect(latencyInfo.frameTiming?.captureToSendLatencyMs).toBeLessThanOrEqual(500)
 
     // Expect jitter buffer/processing delay to be no greater than 500ms on local link
     expect(latencyInfo.averageJitterBufferDelayMs).toBeLessThanOrEqual(500);

Frontend/library/src/PeerConnectionController/LatencyCalculator.ts

@@ -145,20 +145,23 @@ export class LatencyCalculator {
             );
         }
 
-        // If we could not calculate latency because `senderLatencyMs` was missing because of no SenderReport yet
-        // We can try to substitute frame timing capture to send latency instead.
+        // Calculate E2E latency using video-timing capture to send time + one way network latency + receiver-side latency
         if (
-            latencyInfo.senderLatencyMs === undefined &&
             latencyInfo.frameTiming !== undefined &&
-            latencyInfo.frameTiming.captureToSendLatencyMs !== undefined
+            latencyInfo.frameTiming.captureToSendLatencyMs !== undefined &&
+            latencyInfo.averageProcessingDelayMs !== undefined &&
+            latencyInfo.rttMs !== undefined
         ) {
-            latencyInfo.senderLatencyMs = latencyInfo.frameTiming.captureToSendLatencyMs;
+            latencyInfo.averageE2ELatency =
+                latencyInfo.frameTiming.captureToSendLatencyMs +
+                latencyInfo.rttMs * 0.5 +
+                latencyInfo.averageProcessingDelayMs;
         }
 
-        // Calculate E2E latency as sender-side latency + one way network latency + receiver-side latency
+        // Calculate E2E latency as abs-capture-time capture to send latency + one way network latency + receiver-side latency
         if (
-            latencyInfo.averageProcessingDelayMs !== undefined &&
             latencyInfo.senderLatencyMs != undefined &&
+            latencyInfo.averageProcessingDelayMs !== undefined &&
             latencyInfo.rttMs !== undefined
         ) {
             latencyInfo.averageE2ELatency =
@@ -353,12 +356,19 @@ export class LatencyCalculator {
  */
 export class LatencyInfo {
     /**
-     * The time taken from sender frame capture to receiver frame receipt.
+     * The time taken from the moment a frame is done capturing to the moment it is sent over the network.
      * Note: This can only be calculated if both offer and answer contain the
-     * the RTP header extension for `abs-capture-time`.
+     * the RTP header extension for `video-timing` (Chrome only for now)
      */
     public senderLatencyMs: number | undefined = undefined;
 
+    /**
+     * The time taken from the moment a frame is done capturing to the moment it is sent over the network.
+     * Note: This can only be calculated if both offer and answer contain the
+     * the RTP header extension for `abs-capture-time` (Chrome only for now)
+     */
+    public senderLatencyAbsCaptureTimeMs: number | undefined = undefined;
+
     /* The round trip time (milliseconds) between each sender->receiver->sender */
     public rttMs: number | undefined = undefined;
 

Frontend/library/src/PeerConnectionController/PeerConnectionController.ts

@@ -272,7 +272,7 @@ export class PeerConnectionController {
     }
 
     isFirefox(): boolean {
-        return typeof (window as any).InstallTrigger !== 'undefined';
+        return navigator.userAgent.indexOf('Firefox') > 0;
     }
 
     /**

Frontend/ui-library/src/UI/StatsPanel.ts

@@ -414,17 +414,17 @@ export class StatsPanel {
             // Sender latency calculated using timing stats
             if (latencyInfo.frameTiming.captureToSendLatencyMs !== undefined) {
                 this.addOrUpdateLatencyStat(
-                    'CaptureToSend',
-                    'Capture to send latency (ms)',
+                    'VideoTimingCaptureToSend',
+                    'Post-capture to send latency (ms)',
                     Math.ceil(latencyInfo.frameTiming.captureToSendLatencyMs).toString()
                 );
             }
         }
 
         if (latencyInfo.senderLatencyMs !== undefined) {
             this.addOrUpdateLatencyStat(
-                'SenderSideLatency',
-                'Sender latency (ms)',
+                'AbsCaptureTimeToSendLatency',
+                'Post-capture (abs-ct) to send latency (ms)',
                 Math.ceil(latencyInfo.senderLatencyMs).toString()
             );
         }