Add: Latency measurements such as sender to receiver latency measurements using RTP header extensions abs-capture-time and video-timing

.github/workflows/healthcheck-libraries.yml

@@ -57,4 +57,8 @@ jobs:
 
     - name: Clean build of frontend implementation
       working-directory: Frontend/implementations/typescript
-      run: npm install && npm run build
+      run: npm install && npm run build
+
+    - name: Clean build of frontend implementation as ES6
+      working-directory: Frontend/implementations/typescript
+      run: npm install && npm run build:esm

.github/workflows/healthcheck-streaming.yml

@@ -60,11 +60,11 @@ jobs:
       uses: robinraju/release-downloader@v1
       with:
         repository: 'EpicGamesExt/PixelStreamingInfrastructure'
-        tag: 'minimal-streamer'
-        fileName: 'Minimal-PixelStreamer-5.5.7z'
+        tag: 'minimal-streamer-5.5'
+        fileName: 'Minimal-PixelStreamer-5.5-Win64-Development.7z'
 
     - name: Extract streamer
-      run: 7z x -oStreamer Minimal-PixelStreamer-5.5.7z
+      run: 7z x -oStreamer Minimal-PixelStreamer-5.5-Win64-Development.7z
 
     - name: Build Common
       working-directory: Common
@@ -109,7 +109,7 @@ jobs:
 
     - name: Run Streamer
       working-directory: Streamer
-      run: Start-Process ".\Minimal\Binaries\Win64\Minimal-Win64-Shipping-Cmd.exe" -ArgumentList "-warp","-dx12","-windowed","-res=1920","-resy=720","-PixelStreamingURL=ws://localhost:8888","-RenderOffScreen","-AllowSoftwaRerendering","-PixelStreamingEncoderCodec=vp8"
+      run: Start-Process ".\Minimal\Binaries\Win64\Minimal-Cmd.exe" -ArgumentList "-warp","-dx12","-windowed","-resx=1920","-resy=720","-PixelStreamingURL=ws://localhost:8888","-RenderOffScreen","-AllowSoftwareRendering","-PixelStreamingEncoderCodec=vp8", "-Log=Minimal.log"
 
     - name: Prepare test
       working-directory: Extras\MinimalStreamTester
@@ -121,9 +121,14 @@ jobs:
     - name: Wait for signalling to come up
       run: curl --retry 10 --retry-delay 20 --retry-connrefused http://localhost:999/api/status
 
+
     - name: Wait for streamer to come up
       run: curl --retry 10 --retry-delay 20 --retry-connrefused http://localhost:999/api/streamers/DefaultStreamer
 
+    - name: Output streamer logs
+      working-directory: Streamer
+      run: ls ".\Minimal\" && Test-Path ".\Minimal\Saved\Logs\Minimal.log" && cat ".\Minimal\Saved\Logs\Minimal.log"
+
     - name: Test if we can stream
       working-directory: Extras\MinimalStreamTester
       run: |

Common/src/Util/SdpUtils.ts

@@ -0,0 +1,31 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+export class SDPUtils {
+    static addVideoHeaderExtensionToSdp(sdp: string, uri: string): string {
+        // Find the highest used header extension id by sorting the extension ids used,
+        // eliminating duplicates and adding one.
+        // Todo: Update this when WebRTC in Chrome supports the header extension API.
+        const usedIds = sdp
+            .split('\n')
+            .filter((line) => line.startsWith('a=extmap:'))
+            .map((line) => parseInt(line.split(' ')[0].substring(9), 10))
+            .sort((a, b) => a - b)
+            .filter((item, index, array) => array.indexOf(item) === index);
+        const nextId = usedIds[usedIds.length - 1] + 1;
+        const extmapLine = 'a=extmap:' + nextId + ' ' + uri + '\r\n';
+
+        const sections = sdp.split('\nm=').map((part, index) => {
+            return (index > 0 ? 'm=' + part : part).trim() + '\r\n';
+        });
+        const sessionPart = sections.shift();
+        // Only add extension to m=video media section
+        return (
+            sessionPart +
+            sections
+                .map((mediaSection) =>
+                    mediaSection.startsWith('m=video') ? mediaSection + extmapLine : mediaSection
+                )
+                .join('')
+        );
+    }
+}

Common/src/pixelstreamingcommon.ts

@@ -10,3 +10,4 @@ export { MessageRegistry } from './Messages/message_registry';
 export * as Messages from './Messages/signalling_messages';
 export * as MessageHelpers from './Messages/message_helpers';
 export { KeepaliveMonitor } from './Protocol/KeepaliveMonitor';
+export * from './Util/SdpUtils';

Extras/FrontendTests/README.md

@@ -4,13 +4,9 @@
 ### Setup
 ```
 npm install
-npx playwright install --with-deps
-```
-
-The above command should install the required browsers but for some reason I find I have to install chrome manually using the following command.
-
-```
-npx playwright install chrome
+npx playwright install-deps
+npx playwright install firefox
+npx playwright install chromium
 ```
 
 ### Prepare

Extras/FrontendTests/dockerfiles/linux/Dockerfile

@@ -4,8 +4,9 @@ WORKDIR /tester
 COPY /Extras/FrontendTests .
 
 RUN npm install
-RUN npx playwright install --with-deps
-RUN npx playwright install chrome
+RUN npx playwright install firefox
+RUN npx playwright install chromium
+RUN npx playwright install-deps
 
 VOLUME /tester/playwright-report
 

Extras/FrontendTests/package.json

@@ -4,21 +4,20 @@
     "description": "",
     "main": "index.js",
     "scripts": {
-        "test": "playwright test",
-        "build": "",
-        "clean": ""
+        "test": "npx playwright test",
+        "build": ""
     },
     "keywords": [],
     "author": "Epic Games",
     "license": "MIT",
     "devDependencies": {
-        "@playwright/test": "^1.49.0",
+        "@playwright/test": "^1.49.1",
         "@types/node": "^20.12.7",
         "@types/uuid": "^9.0.8"
     },
     "dependencies": {
-        "@epicgames-ps/lib-pixelstreamingfrontend-ue5.5": "*",
         "@epicgames-ps/js-streamer": "^0.0.4",
+        "@epicgames-ps/lib-pixelstreamingfrontend-ue5.5": "*",
         "dotenv": "^16.4.5",
         "node-fetch": "^2.7.0",
         "uuid": "^9.0.0"

Extras/FrontendTests/playwright.config.ts

@@ -14,8 +14,8 @@ export default defineConfig({
   forbidOnly: !!process.env.CI,
   /* Retry on CI only */
   retries: process.env.CI ? 2 : 0,
-  /* Opt out of parallel tests on CI. */
-  workers: process.env.CI ? 1 : 5,
+  /* Opt out of parallel tests in general as multiple streamers mean they can connect to the wrong test */
+  workers: 1,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: [['html', { open: 'never' }]],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
@@ -31,7 +31,7 @@ export default defineConfig({
   projects: [
     {
       name: 'chrome',
-      use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+      use: { ...devices['Desktop Chrome'], channel: 'chromium' },
     },
     // {
     //   name: 'chromium',

Extras/FrontendTests/tests/basic_stream.spec.ts

@@ -1,6 +1,7 @@
 import { test } from './fixtures';
 import { expect } from './matchers';
 import * as helpers from './helpers';
+import { StatsReceivedEvent } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.5';
 
 // NOTE: add a new test to check qp values
 
@@ -10,20 +11,22 @@ test('Test default stream.', {
 }, async ({ page, streamerId }) => {
 
     await page.goto(`/?StreamerId=${streamerId}`);
-    await page.getByText('Click to start').click();
 
     // let the stream run for a short duration
-    await helpers.waitForVideo(page);
-    await helpers.delay(1000);
-
-    // query the frontend for its calculated stats
-    const frame_count:number = await page.evaluate(()=> {
-        let videoStats = pixelStreaming._webRtcController.peerConnectionController.aggregatedStats.inboundVideoStats;
-        return videoStats.framesReceived;
+    await helpers.startAndWaitForVideo(page);
+
+    let frameCount: number = await page.evaluate(()=> {
+        return new Promise<number>((resolve) => {
+            window.pixelStreaming.addEventListener("statsReceived", (e: StatsReceivedEvent) => {
+                if(e.data.aggregatedStats && e.data.aggregatedStats.inboundVideoStats && e.data.aggregatedStats.inboundVideoStats.framesReceived) {
+                    resolve(e.data.aggregatedStats.inboundVideoStats.framesReceived);
+                }
+            });
+        });
     });
 
     // pass the test if we recorded any frames
-    expect(frame_count).toBeGreaterThan(0);
+    expect(frameCount).toBeGreaterThan(0);
 });