From c50d8b03aaa141fabbc91e1b159c87d961c1cfa4 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 4 Sep 2023 11:44:37 +0200 Subject: [PATCH 1/7] Revert "Disable opus RED when using E2EE (#858)" This reverts commit 644db17718c5edb174287c86ada4a1c06c33dd4e. --- .changeset/gorgeous-cameras-peel.md | 5 ----- src/room/participant/LocalParticipant.ts | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 .changeset/gorgeous-cameras-peel.md diff --git a/.changeset/gorgeous-cameras-peel.md b/.changeset/gorgeous-cameras-peel.md deleted file mode 100644 index 4c0484ba5f..0000000000 --- a/.changeset/gorgeous-cameras-peel.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"livekit-client": patch ---- - -Disable opus RED when using E2EE diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index b9e81fd797..55de1fb395 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -649,7 +649,7 @@ export default class LocalParticipant extends Participant { disableDtx: !(opts.dtx ?? true), encryption: this.encryptionType, stereo: isStereo, - disableRed: this.isE2EEEnabled || !(opts.red ?? true), + disableRed: !(opts.red ?? true), }); // compute encodings and layers for video From c89ec6e5b9031eabbcf9113741fa2b59348a09fa Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 4 Sep 2023 17:05:52 +0200 Subject: [PATCH 2/7] wip --- package.json | 1 + src/e2ee/constants.ts | 1 + src/e2ee/worker/FrameCryptor.ts | 177 ++++++++++++++++++++++++++++---- yarn.lock | 5 + 4 files changed, 166 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index 08e992b4a0..b701160424 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "@bufbuild/protobuf": "^1.3.0", "events": "^3.3.0", "loglevel": "^1.8.0", + "opus-red-parser": "^1.0.0", "sdp-transform": "^2.14.1", "ts-debounce": "^4.0.0", "typed-emitter": "^2.1.0", diff --git a/src/e2ee/constants.ts b/src/e2ee/constants.ts index 9c541ab4dc..f7ed609249 100644 --- a/src/e2ee/constants.ts +++ b/src/e2ee/constants.ts @@ -24,6 +24,7 @@ export const UNENCRYPTED_BYTES = { key: 10, delta: 3, audio: 1, // frame.type is not set on audio, so this is set manually + red1: 5, // opus red with 1 redundancy frame empty: 0, } as const; diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index e7f312c299..809424d3d1 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ // TODO code inspired by https://github.com/webrtc/samples/blob/gh-pages/src/content/insertable-streams/endtoend-encryption/js/worker.js import { EventEmitter } from 'events'; +import * as red from 'opus-red-parser'; import type TypedEventEmitter from 'typed-emitter'; import { workerLogger } from '../../logger'; import type { VideoCodec } from '../../room/track/options'; @@ -233,29 +234,111 @@ export class FrameCryptor extends BaseFrameCryptor { // payload |IV...(length = IV_LENGTH)|R|IV_LENGTH|KID | // ---------+-------------------------+-+---------+---- try { - const cipherText = await crypto.subtle.encrypt( - { - name: ENCRYPTION_ALGORITHM, + // @ts-expect-error not supported in safari + if (encodedFrame.getMetadata().payloadType === 63) { + const opusRedFrame = red.split(encodedFrame.data); + + const primaryFrameCipher = await crypto.subtle.encrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: Uint8Array.from(opusRedFrame.primaryBlock.data.slice(0, 1)), + }, + encryptionKey, + Uint8Array.from(opusRedFrame.primaryBlock.data.slice(1)), + ); + const primaryBuffer = new ArrayBuffer(primaryFrameCipher.byteLength + 1); + const primaryData = new Uint8Array(primaryBuffer); + primaryData.set(opusRedFrame.primaryBlock.data.slice(0, 1), 0); + primaryData.set(new Uint8Array(primaryFrameCipher), 1); + opusRedFrame.primaryBlock.data = primaryData; + const redundancyBlocksEncrypted = await Promise.all( + opusRedFrame.redundancyBlocks.map(async (block) => { + const cipherText = await crypto.subtle.encrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: block.data.slice(0, 1), + }, + encryptionKey, + block.data.slice(1), + ); + block.data = new Uint8Array(cipherText); + block.header.blockLength = cipherText.byteLength; + return block; + }), + ); + + console.log( + 'encrypting opus red', + { + primary: opusRedFrame.primaryBlock, + redundancyBlocksEncrypted, + }, iv, - additionalData: new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength), - }, - encryptionKey, - new Uint8Array(encodedFrame.data, this.getUnencryptedBytes(encodedFrame)), - ); + ); - const newData = new ArrayBuffer( - frameHeader.byteLength + cipherText.byteLength + iv.byteLength + frameTrailer.byteLength, - ); - const newUint8 = new Uint8Array(newData); + const encryptedRed = red.join({ + primaryBlock: opusRedFrame.primaryBlock, + redundancyBlocks: redundancyBlocksEncrypted, + }); + + const newData = new ArrayBuffer( + encryptedRed.byteLength + iv.byteLength + frameTrailer.byteLength, + ); + const newUint8 = new Uint8Array(newData); + newUint8.set(opusRedFrame.primaryBlock.data.slice(0, 1), 0); // add ciphertext. + newUint8.set(new Uint8Array(encryptedRed), 1); // add ciphertext. + newUint8.set(new Uint8Array(iv), encryptedRed.byteLength); // append IV. + newUint8.set(frameTrailer, encryptedRed.byteLength + iv.byteLength); // append frame trailer. + + //*** DEBUG DECRYPT DIRECTLY */ + + console.log('split before decrypt', opusRedFrame, iv); + const primaryDecrypted = await crypto.subtle.decrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: Uint8Array.from(opusRedFrame.primaryBlock.data.slice(0, 1)), + }, + keySet!.encryptionKey, + Uint8Array.from(opusRedFrame.primaryBlock.data.slice(1)), + ); + console.log('Decrypted primary'); - newUint8.set(frameHeader); // copy first bytes. - newUint8.set(new Uint8Array(cipherText), frameHeader.byteLength); // add ciphertext. - newUint8.set(new Uint8Array(iv), frameHeader.byteLength + cipherText.byteLength); // append IV. - newUint8.set(frameTrailer, frameHeader.byteLength + cipherText.byteLength + iv.byteLength); // append frame trailer. + encodedFrame.data = newData; + return controller.enqueue(encodedFrame); + } else { + const cipherText = await crypto.subtle.encrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength), + }, + encryptionKey, + new Uint8Array(encodedFrame.data, this.getUnencryptedBytes(encodedFrame)), + ); - encodedFrame.data = newData; + const newData = new ArrayBuffer( + frameHeader.byteLength + + cipherText.byteLength + + iv.byteLength + + frameTrailer.byteLength, + ); + const newUint8 = new Uint8Array(newData); - return controller.enqueue(encodedFrame); + newUint8.set(frameHeader); // copy first bytes. + newUint8.set(new Uint8Array(cipherText), frameHeader.byteLength); // add ciphertext. + newUint8.set(new Uint8Array(iv), frameHeader.byteLength + cipherText.byteLength); // append IV. + newUint8.set( + frameTrailer, + frameHeader.byteLength + cipherText.byteLength + iv.byteLength, + ); // append frame trailer. + + encodedFrame.data = newData; + + return controller.enqueue(encodedFrame); + } } catch (e: any) { // TODO: surface this to the app. workerLogger.error(e); @@ -301,6 +384,7 @@ export class FrameCryptor extends BaseFrameCryptor { } const data = new Uint8Array(encodedFrame.data); const keyIndex = data[encodedFrame.data.byteLength - 1]; + console.log('key index', keyIndex, data[encodedFrame.data.byteLength - 2]); if (this.keys.getKeySet(keyIndex) && this.keys.hasValidKey) { try { @@ -370,6 +454,58 @@ export class FrameCryptor extends BaseFrameCryptor { ivLength, ); + // @ts-expect-error no support in safari + if (encodedFrame.getMetadata().payloadType === 63) { + const opusRedFrame = red.split( + encodedFrame.data.slice(0, encodedFrame.data.byteLength - frameTrailer.length - ivLength), + ); + console.log('split before decrypt', opusRedFrame, iv); + const primaryDecrypted = await crypto.subtle.decrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: opusRedFrame.primaryBlock.data.slice(0, 1), + }, + ratchetOpts.encryptionKey ?? keySet!.encryptionKey, + opusRedFrame.primaryBlock.data.slice(1), + ); + console.log('Decrypted primary'); + + opusRedFrame.primaryBlock.data = new Uint8Array(primaryDecrypted); + + const redundancyDecrypted = await Promise.all( + opusRedFrame.redundancyBlocks.map(async (block) => { + const plainText = await crypto.subtle.decrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: block.data.slice(0, 1), + }, + ratchetOpts.encryptionKey ?? keySet!.encryptionKey, + block.data.slice(1), + ); + + block.data = new Uint8Array(plainText); + block.header.blockLength = plainText.byteLength; + + return block; + }), + ); + + console.log('decrypted frame', { + primary: opusRedFrame.primaryBlock, + redundancyDecrypted, + }); + + const decryptedFrame = red.join({ + primaryBlock: opusRedFrame.primaryBlock, + redundancyBlocks: redundancyDecrypted, + }); + + encodedFrame.data = decryptedFrame; + return encodedFrame; + } + const cipherTextStart = frameHeader.byteLength; const cipherTextLength = encodedFrame.data.byteLength - @@ -395,6 +531,7 @@ export class FrameCryptor extends BaseFrameCryptor { return encodedFrame; } catch (error: any) { + throw error; if (this.keyProviderOptions.ratchetWindowSize > 0) { if (ratchetOpts.ratchetCount < this.keyProviderOptions.ratchetWindowSize) { workerLogger.debug( @@ -531,6 +668,10 @@ export class FrameCryptor extends BaseFrameCryptor { return UNENCRYPTED_BYTES[frame.type]; } else { + // @ts-expect-error payload type not supported on safari + if (frame.getMetadata().payloadType === 63) { + return UNENCRYPTED_BYTES.red1; + } return UNENCRYPTED_BYTES.audio; } } diff --git a/yarn.lock b/yarn.lock index 817ce5ba89..04583e68bf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4696,6 +4696,11 @@ optionator@^0.9.3: prelude-ls "^1.2.1" type-check "^0.4.0" +opus-red-parser@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/opus-red-parser/-/opus-red-parser-1.0.0.tgz#4007fb0cfb8c94c6422cfb73158025897377de8b" + integrity sha512-bPek9U/wYf5ki1znILBxL9bxBRAOUGL7E7LUNtqIh58lD4ly4BNQjsqfiZyunZ0Lm46k/gfJzvYQB3Fed6pBQw== + os-tmpdir@~1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz" From 813efefea640a1f9dfe79fe7668c13cb3e4d69cb Mon Sep 17 00:00:00 2001 From: lukasIO Date: Mon, 4 Sep 2023 17:40:32 +0200 Subject: [PATCH 3/7] abstract encrypt and decrypt functions --- src/e2ee/worker/FrameCryptor.ts | 189 +++++++++++++++----------------- 1 file changed, 91 insertions(+), 98 deletions(-) diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index 809424d3d1..0c521f9b05 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -236,106 +236,49 @@ export class FrameCryptor extends BaseFrameCryptor { try { // @ts-expect-error not supported in safari if (encodedFrame.getMetadata().payloadType === 63) { - const opusRedFrame = red.split(encodedFrame.data); - - const primaryFrameCipher = await crypto.subtle.encrypt( - { - name: ENCRYPTION_ALGORITHM, - iv, - additionalData: Uint8Array.from(opusRedFrame.primaryBlock.data.slice(0, 1)), - }, + const { primaryBlock, redundancyBlocks } = red.split(encodedFrame.data); + + const primaryBlockEncrypted = await this.encrypt( + primaryBlock.data.slice(1), + primaryBlock.data.slice(0, 1), + frameTrailer, + iv, encryptionKey, - Uint8Array.from(opusRedFrame.primaryBlock.data.slice(1)), ); - const primaryBuffer = new ArrayBuffer(primaryFrameCipher.byteLength + 1); - const primaryData = new Uint8Array(primaryBuffer); - primaryData.set(opusRedFrame.primaryBlock.data.slice(0, 1), 0); - primaryData.set(new Uint8Array(primaryFrameCipher), 1); - opusRedFrame.primaryBlock.data = primaryData; + primaryBlock.data = primaryBlockEncrypted; + const redundancyBlocksEncrypted = await Promise.all( - opusRedFrame.redundancyBlocks.map(async (block) => { - const cipherText = await crypto.subtle.encrypt( - { - name: ENCRYPTION_ALGORITHM, - iv, - additionalData: block.data.slice(0, 1), - }, - encryptionKey, + redundancyBlocks.map(async (block) => { + const blockEncrypted = await this.encrypt( block.data.slice(1), + block.data.slice(0, 1), + frameTrailer, + iv, + encryptionKey, ); - block.data = new Uint8Array(cipherText); - block.header.blockLength = cipherText.byteLength; + block.data = blockEncrypted; + block.header.blockLength = blockEncrypted.length; return block; }), ); - console.log( - 'encrypting opus red', - { - primary: opusRedFrame.primaryBlock, - redundancyBlocksEncrypted, - }, - iv, - ); - const encryptedRed = red.join({ - primaryBlock: opusRedFrame.primaryBlock, + primaryBlock, redundancyBlocks: redundancyBlocksEncrypted, }); - const newData = new ArrayBuffer( - encryptedRed.byteLength + iv.byteLength + frameTrailer.byteLength, - ); - const newUint8 = new Uint8Array(newData); - newUint8.set(opusRedFrame.primaryBlock.data.slice(0, 1), 0); // add ciphertext. - newUint8.set(new Uint8Array(encryptedRed), 1); // add ciphertext. - newUint8.set(new Uint8Array(iv), encryptedRed.byteLength); // append IV. - newUint8.set(frameTrailer, encryptedRed.byteLength + iv.byteLength); // append frame trailer. - - //*** DEBUG DECRYPT DIRECTLY */ - - console.log('split before decrypt', opusRedFrame, iv); - const primaryDecrypted = await crypto.subtle.decrypt( - { - name: ENCRYPTION_ALGORITHM, - iv, - additionalData: Uint8Array.from(opusRedFrame.primaryBlock.data.slice(0, 1)), - }, - keySet!.encryptionKey, - Uint8Array.from(opusRedFrame.primaryBlock.data.slice(1)), - ); - console.log('Decrypted primary'); - - encodedFrame.data = newData; + encodedFrame.data = encryptedRed; return controller.enqueue(encodedFrame); } else { - const cipherText = await crypto.subtle.encrypt( - { - name: ENCRYPTION_ALGORITHM, - iv, - additionalData: new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength), - }, - encryptionKey, + const encryptedFrame = await this.encrypt( new Uint8Array(encodedFrame.data, this.getUnencryptedBytes(encodedFrame)), - ); - - const newData = new ArrayBuffer( - frameHeader.byteLength + - cipherText.byteLength + - iv.byteLength + - frameTrailer.byteLength, - ); - const newUint8 = new Uint8Array(newData); - - newUint8.set(frameHeader); // copy first bytes. - newUint8.set(new Uint8Array(cipherText), frameHeader.byteLength); // add ciphertext. - newUint8.set(new Uint8Array(iv), frameHeader.byteLength + cipherText.byteLength); // append IV. - newUint8.set( + new Uint8Array(encodedFrame.data, 0, this.getUnencryptedBytes(encodedFrame)), frameTrailer, - frameHeader.byteLength + cipherText.byteLength + iv.byteLength, - ); // append frame trailer. + iv, + encryptionKey, + ); - encodedFrame.data = newData; + encodedFrame.data = encryptedFrame.buffer; return controller.enqueue(encodedFrame); } @@ -351,6 +294,43 @@ export class FrameCryptor extends BaseFrameCryptor { } } + private async encrypt( + payload: Uint8Array, + unencryptedHeaderBytes: Uint8Array, + frameTrailer: Uint8Array, + iv: ArrayBuffer, + key: CryptoKey, + ) { + const cipherText = await crypto.subtle.encrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: unencryptedHeaderBytes, + }, + key, + payload, + ); + const newData = new ArrayBuffer( + unencryptedHeaderBytes.byteLength + + cipherText.byteLength + + iv.byteLength + + frameTrailer.byteLength, + ); + const encryptedBytesWithTrailer = new Uint8Array(newData); + + encryptedBytesWithTrailer.set(unencryptedHeaderBytes); // copy first bytes. + encryptedBytesWithTrailer.set(new Uint8Array(cipherText), unencryptedHeaderBytes.byteLength); // add ciphertext. + encryptedBytesWithTrailer.set( + new Uint8Array(iv), + unencryptedHeaderBytes.byteLength + cipherText.byteLength, + ); // append IV. + encryptedBytesWithTrailer.set( + frameTrailer, + unencryptedHeaderBytes.byteLength + cipherText.byteLength + iv.byteLength, + ); // append frame trailer. + return encryptedBytesWithTrailer; + } + /** * Function that will be injected in a stream and will decrypt the given encoded frames. * @@ -506,28 +486,18 @@ export class FrameCryptor extends BaseFrameCryptor { return encodedFrame; } - const cipherTextStart = frameHeader.byteLength; const cipherTextLength = encodedFrame.data.byteLength - (frameHeader.byteLength + ivLength + frameTrailer.byteLength); - const plainText = await crypto.subtle.decrypt( - { - name: ENCRYPTION_ALGORITHM, - iv, - additionalData: new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength), - }, - ratchetOpts.encryptionKey ?? keySet!.encryptionKey, - new Uint8Array(encodedFrame.data, cipherTextStart, cipherTextLength), + const decryptedData = await this.decrypt( + new Uint8Array(encodedFrame.data, frameHeader.byteLength, cipherTextLength), + new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength), + iv, + ratchetOpts?.encryptionKey ?? keySet!.encryptionKey, ); - const newData = new ArrayBuffer(frameHeader.byteLength + plainText.byteLength); - const newUint8 = new Uint8Array(newData); - - newUint8.set(new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength)); - newUint8.set(new Uint8Array(plainText), frameHeader.byteLength); - - encodedFrame.data = newData; + encodedFrame.data = decryptedData.buffer; return encodedFrame; } catch (error: any) { @@ -586,6 +556,29 @@ export class FrameCryptor extends BaseFrameCryptor { } } + private async decrypt( + payload: Uint8Array, + unencryptedHeaderBytes: Uint8Array, + iv: ArrayBuffer, + key: CryptoKey, + ) { + const plainText = await crypto.subtle.decrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: unencryptedHeaderBytes, + }, + key, + payload, + ); + + const buffer = new ArrayBuffer(unencryptedHeaderBytes.byteLength + plainText.byteLength); + const decryptedBytes = new Uint8Array(buffer); + decryptedBytes.set(new Uint8Array(unencryptedHeaderBytes, 0)); + decryptedBytes.set(new Uint8Array(plainText), unencryptedHeaderBytes.byteLength); + return decryptedBytes; + } + /** * Construct the IV used for AES-GCM and sent (in plain) with the packet similar to * https://tools.ietf.org/html/rfc7714#section-8.1 From 349df09e1f923ee46e3e2001d3320e01f0aeca5f Mon Sep 17 00:00:00 2001 From: lukasIO Date: Tue, 5 Sep 2023 11:33:25 +0200 Subject: [PATCH 4/7] primary block decryption working --- src/e2ee/worker/FrameCryptor.ts | 92 +++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 40 deletions(-) diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index 0c521f9b05..ef002e7462 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -267,6 +267,8 @@ export class FrameCryptor extends BaseFrameCryptor { redundancyBlocks: redundancyBlocksEncrypted, }); + console.log('primary block', primaryBlock.data); + encodedFrame.data = encryptedRed; return controller.enqueue(encodedFrame); } else { @@ -420,71 +422,81 @@ export class FrameCryptor extends BaseFrameCryptor { // ---------+-------------------------+-+---------+---- try { - const frameHeader = new Uint8Array( - encodedFrame.data, - 0, - this.getUnencryptedBytes(encodedFrame), - ); - const frameTrailer = new Uint8Array(encodedFrame.data, encodedFrame.data.byteLength - 2, 2); - - const ivLength = frameTrailer[0]; - const iv = new Uint8Array( - encodedFrame.data, - encodedFrame.data.byteLength - ivLength - frameTrailer.byteLength, - ivLength, - ); - // @ts-expect-error no support in safari if (encodedFrame.getMetadata().payloadType === 63) { - const opusRedFrame = red.split( - encodedFrame.data.slice(0, encodedFrame.data.byteLength - frameTrailer.length - ivLength), + const { primaryBlock, redundancyBlocks } = red.split(encodedFrame.data); + + const primaryFrameTrailer = primaryBlock.data.slice(primaryBlock.data.byteLength - 2); + const ivLength = primaryFrameTrailer[0]; + + const primaryFramePayload = primaryBlock.data.slice( + 0, + primaryBlock.data.byteLength - 2 - ivLength, ); - console.log('split before decrypt', opusRedFrame, iv); - const primaryDecrypted = await crypto.subtle.decrypt( - { - name: ENCRYPTION_ALGORITHM, - iv, - additionalData: opusRedFrame.primaryBlock.data.slice(0, 1), - }, - ratchetOpts.encryptionKey ?? keySet!.encryptionKey, - opusRedFrame.primaryBlock.data.slice(1), + + console.log('frame trailer', primaryFrameTrailer, primaryBlock.data); + + const primaryKey = + ratchetOpts.encryptionKey ?? this.keys.getKeySet(primaryFrameTrailer[1])?.encryptionKey; + if (!primaryKey) { + throw new TypeError( + `key missing for primary opus frame at index ${primaryFrameTrailer[1]}`, + ); + } + const iv = primaryBlock.data.slice( + primaryBlock.data.byteLength - ivLength - primaryFrameTrailer.byteLength, + primaryBlock.data.byteLength - primaryFrameTrailer.byteLength, + ); + const primaryDecrypted = await this.decrypt( + primaryFramePayload.slice(1), + primaryFramePayload.slice(0, 1), + iv, + primaryKey, ); + primaryBlock.data = primaryDecrypted; console.log('Decrypted primary'); - opusRedFrame.primaryBlock.data = new Uint8Array(primaryDecrypted); - const redundancyDecrypted = await Promise.all( - opusRedFrame.redundancyBlocks.map(async (block) => { - const plainText = await crypto.subtle.decrypt( - { - name: ENCRYPTION_ALGORITHM, - iv, - additionalData: block.data.slice(0, 1), - }, - ratchetOpts.encryptionKey ?? keySet!.encryptionKey, + redundancyBlocks.map(async (block) => { + const decryptedBlock = await this.decrypt( block.data.slice(1), + block.data.slice(0, 1), + iv, + primaryKey, ); - - block.data = new Uint8Array(plainText); - block.header.blockLength = plainText.byteLength; + block.data = decryptedBlock; + block.header.blockLength = decryptedBlock.byteLength; return block; }), ); console.log('decrypted frame', { - primary: opusRedFrame.primaryBlock, + primary: primaryBlock, redundancyDecrypted, }); const decryptedFrame = red.join({ - primaryBlock: opusRedFrame.primaryBlock, + primaryBlock: primaryBlock, redundancyBlocks: redundancyDecrypted, }); encodedFrame.data = decryptedFrame; return encodedFrame; } + const frameHeader = new Uint8Array( + encodedFrame.data, + 0, + this.getUnencryptedBytes(encodedFrame), + ); + const frameTrailer = new Uint8Array(encodedFrame.data, encodedFrame.data.byteLength - 2, 2); + + const ivLength = frameTrailer[0]; + const iv = new Uint8Array( + encodedFrame.data, + encodedFrame.data.byteLength - ivLength - frameTrailer.byteLength, + ivLength, + ); const cipherTextLength = encodedFrame.data.byteLength - From 32f7883d334e553be46b66c302c58cd3891d053f Mon Sep 17 00:00:00 2001 From: lukasIO Date: Tue, 5 Sep 2023 12:38:36 +0200 Subject: [PATCH 5/7] wip red audio codec detection --- src/e2ee/E2eeManager.ts | 17 ++-- src/e2ee/types.ts | 12 +-- src/e2ee/utils.ts | 14 ++-- src/e2ee/worker/FrameCryptor.ts | 139 ++++++++++++++------------------ src/e2ee/worker/e2ee.worker.ts | 2 +- src/room/PCTransport.ts | 3 + src/room/RTCEngine.ts | 15 +++- src/room/events.ts | 1 + src/room/track/options.ts | 4 + src/room/utils.ts | 6 +- 10 files changed, 111 insertions(+), 102 deletions(-) diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index cc0fdac2c0..0b3d75aaed 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -10,7 +10,7 @@ import { EngineEvent, ParticipantEvent, RoomEvent } from '../room/events'; import LocalTrack from '../room/track/LocalTrack'; import type RemoteTrack from '../room/track/RemoteTrack'; import type { Track } from '../room/track/Track'; -import type { VideoCodec } from '../room/track/options'; +import type { AudioCodec, VideoCodec } from '../room/track/options'; import type { BaseKeyProvider } from './KeyProvider'; import { E2EE_FLAG } from './constants'; import { type E2EEManagerCallbacks, EncryptionEvent, KeyProviderEvent } from './events'; @@ -21,14 +21,14 @@ import type { EncodeMessage, InitMessage, KeyInfo, - RTPVideoMapMessage, + RTPMapMessage, RatchetRequestMessage, RemoveTransformMessage, SetKeyMessage, SifTrailerMessage, UpdateCodecMessage, } from './types'; -import { isE2EESupported, isScriptTransformSupported, mimeTypeToVideoCodecString } from './utils'; +import { isE2EESupported, isScriptTransformSupported, mimeTypeToCodecString } from './utils'; /** * @experimental @@ -155,6 +155,9 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter { this.postRTPMap(rtpMap); }); + engine.on(EngineEvent.RTPAudioMapUpdate, (rtpMap) => { + this.postRTPMap(rtpMap); + }); } private setupEventListeners(room: Room, keyProvider: BaseKeyProvider) { @@ -258,14 +261,14 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter) { + private postRTPMap(map: Map | Map) { if (!this.worker) { throw TypeError('could not post rtp map, worker is missing'); } if (!this.room?.localParticipant.identity) { throw TypeError('could not post rtp map, local participant identity is missing'); } - const msg: RTPVideoMapMessage = { + const msg: RTPMapMessage = { kind: 'setRTPMap', data: { map, @@ -299,7 +302,7 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter; + map: Map | Map; participantIdentity: string; }; } @@ -45,7 +45,7 @@ export interface EncodeMessage extends BaseMessage { readableStream: ReadableStream; writableStream: WritableStream; trackId: string; - codec?: VideoCodec; + codec?: VideoCodec | AudioCodec; }; } @@ -62,7 +62,7 @@ export interface UpdateCodecMessage extends BaseMessage { data: { participantIdentity: string; trackId: string; - codec: VideoCodec; + codec: VideoCodec | AudioCodec; }; } @@ -112,7 +112,7 @@ export type E2EEWorkerMessage = | ErrorMessage | EnableMessage | RemoveTransformMessage - | RTPVideoMapMessage + | RTPMapMessage | UpdateCodecMessage | RatchetRequestMessage | RatchetMessage diff --git a/src/e2ee/utils.ts b/src/e2ee/utils.ts index 31ad917104..0a11e469f1 100644 --- a/src/e2ee/utils.ts +++ b/src/e2ee/utils.ts @@ -1,5 +1,5 @@ -import { videoCodecs } from '../room/track/options'; -import type { VideoCodec } from '../room/track/options'; +import type { AudioCodec, VideoCodec } from '../room/track/options'; +import { isAudioCodec, isVideoCodec } from '../room/utils'; import { ENCRYPTION_ALGORITHM } from './constants'; export function isE2EESupported() { @@ -116,12 +116,12 @@ export function createE2EEKey(): Uint8Array { return window.crypto.getRandomValues(new Uint8Array(32)); } -export function mimeTypeToVideoCodecString(mimeType: string) { - const codec = mimeType.split('/')[1].toLowerCase() as VideoCodec; - if (!videoCodecs.includes(codec)) { - throw Error(`Video codec not supported: ${codec}`); +export function mimeTypeToCodecString(mimeType: string) { + const codec = mimeType.split('/')[1].toLowerCase() as VideoCodec | AudioCodec; + if (isVideoCodec(codec) || isAudioCodec(codec)) { + return codec; } - return codec; + throw Error(`Codec not supported: ${codec}`); } /** diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index ef002e7462..f270d1b88c 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -4,7 +4,7 @@ import { EventEmitter } from 'events'; import * as red from 'opus-red-parser'; import type TypedEventEmitter from 'typed-emitter'; import { workerLogger } from '../../logger'; -import type { VideoCodec } from '../../room/track/options'; +import type { AudioCodec, VideoCodec } from '../../room/track/options'; import { ENCRYPTION_ALGORITHM, IV_LENGTH, UNENCRYPTED_BYTES } from '../constants'; import { CryptorError, CryptorErrorReason } from '../errors'; import { CryptorCallbacks, CryptorEvent } from '../events'; @@ -55,9 +55,9 @@ export class FrameCryptor extends BaseFrameCryptor { private keys: ParticipantKeyHandler; - private videoCodec?: VideoCodec; + private codec?: VideoCodec | AudioCodec; - private rtpMap: Map; + private rtpMap: Map; private keyProviderOptions: KeyProviderOptions; @@ -117,18 +117,18 @@ export class FrameCryptor extends BaseFrameCryptor { } /** - * Update the video codec used by the mediaStreamTrack + * Update the codec used by the mediaStreamTrack * @param codec */ - setVideoCodec(codec: VideoCodec) { - this.videoCodec = codec; + setCodec(codec: VideoCodec | AudioCodec) { + this.codec = codec; } /** * rtp payload type map used for figuring out codec of payload type when encoding * @param map */ - setRtpMap(map: Map) { + setRtpMap(map: Map) { this.rtpMap = map; } @@ -137,11 +137,11 @@ export class FrameCryptor extends BaseFrameCryptor { readable: ReadableStream, writable: WritableStream, trackId: string, - codec?: VideoCodec, + codec?: VideoCodec | AudioCodec, ) { if (codec) { workerLogger.info('setting codec on cryptor to', { codec }); - this.videoCodec = codec; + this.codec = codec; } const transformFn = operation === 'encode' ? this.encodeFunction : this.decodeFunction; @@ -213,13 +213,6 @@ export class FrameCryptor extends BaseFrameCryptor { encodedFrame.timestamp, ); - // Thіs is not encrypted and contains the VP8 payload descriptor or the Opus TOC byte. - const frameHeader = new Uint8Array( - encodedFrame.data, - 0, - this.getUnencryptedBytes(encodedFrame), - ); - // Frame trailer contains the R|IV_LENGTH and key index const frameTrailer = new Uint8Array(2); @@ -234,8 +227,7 @@ export class FrameCryptor extends BaseFrameCryptor { // payload |IV...(length = IV_LENGTH)|R|IV_LENGTH|KID | // ---------+-------------------------+-+---------+---- try { - // @ts-expect-error not supported in safari - if (encodedFrame.getMetadata().payloadType === 63) { + if (this.getCodec(encodedFrame) === 'red') { const { primaryBlock, redundancyBlocks } = red.split(encodedFrame.data); const primaryBlockEncrypted = await this.encrypt( @@ -267,8 +259,6 @@ export class FrameCryptor extends BaseFrameCryptor { redundancyBlocks: redundancyBlocksEncrypted, }); - console.log('primary block', primaryBlock.data); - encodedFrame.data = encryptedRed; return controller.enqueue(encodedFrame); } else { @@ -366,7 +356,6 @@ export class FrameCryptor extends BaseFrameCryptor { } const data = new Uint8Array(encodedFrame.data); const keyIndex = data[encodedFrame.data.byteLength - 1]; - console.log('key index', keyIndex, data[encodedFrame.data.byteLength - 2]); if (this.keys.getKeySet(keyIndex) && this.keys.hasValidKey) { try { @@ -422,47 +411,41 @@ export class FrameCryptor extends BaseFrameCryptor { // ---------+-------------------------+-+---------+---- try { - // @ts-expect-error no support in safari - if (encodedFrame.getMetadata().payloadType === 63) { + if (this.getCodec(encodedFrame) === 'red') { const { primaryBlock, redundancyBlocks } = red.split(encodedFrame.data); - const primaryFrameTrailer = primaryBlock.data.slice(primaryBlock.data.byteLength - 2); - const ivLength = primaryFrameTrailer[0]; - - const primaryFramePayload = primaryBlock.data.slice( - 0, - primaryBlock.data.byteLength - 2 - ivLength, - ); - - console.log('frame trailer', primaryFrameTrailer, primaryBlock.data); + const primaryFrame = this.extractFrameInfo(primaryBlock.data); const primaryKey = - ratchetOpts.encryptionKey ?? this.keys.getKeySet(primaryFrameTrailer[1])?.encryptionKey; + ratchetOpts.encryptionKey ?? this.keys.getKeySet(primaryFrame.keyIndex)?.encryptionKey; if (!primaryKey) { throw new TypeError( - `key missing for primary opus frame at index ${primaryFrameTrailer[1]}`, + `key missing for primary opus frame at index ${primaryFrame.keyIndex}`, ); } - const iv = primaryBlock.data.slice( - primaryBlock.data.byteLength - ivLength - primaryFrameTrailer.byteLength, - primaryBlock.data.byteLength - primaryFrameTrailer.byteLength, - ); + const primaryDecrypted = await this.decrypt( - primaryFramePayload.slice(1), - primaryFramePayload.slice(0, 1), - iv, + primaryFrame.encryptedPayload.slice(1), + primaryFrame.encryptedPayload.slice(0, 1), + primaryFrame.iv, primaryKey, ); primaryBlock.data = primaryDecrypted; - console.log('Decrypted primary'); const redundancyDecrypted = await Promise.all( redundancyBlocks.map(async (block) => { + const redundancyFrame = this.extractFrameInfo(block.data); + const key = this.keys.getKeySet(redundancyFrame.keyIndex)?.encryptionKey; + if (!key) { + throw new TypeError( + `key missing for redundancy opus frame at index ${primaryFrame.keyIndex}`, + ); + } const decryptedBlock = await this.decrypt( - block.data.slice(1), - block.data.slice(0, 1), - iv, - primaryKey, + redundancyFrame.encryptedPayload.slice(1), + redundancyFrame.encryptedPayload.slice(0, 1), + redundancyFrame.iv, + key, ); block.data = decryptedBlock; block.header.blockLength = decryptedBlock.byteLength; @@ -471,11 +454,6 @@ export class FrameCryptor extends BaseFrameCryptor { }), ); - console.log('decrypted frame', { - primary: primaryBlock, - redundancyDecrypted, - }); - const decryptedFrame = red.join({ primaryBlock: primaryBlock, redundancyBlocks: redundancyDecrypted, @@ -484,28 +462,14 @@ export class FrameCryptor extends BaseFrameCryptor { encodedFrame.data = decryptedFrame; return encodedFrame; } - const frameHeader = new Uint8Array( - encodedFrame.data, - 0, - this.getUnencryptedBytes(encodedFrame), - ); - const frameTrailer = new Uint8Array(encodedFrame.data, encodedFrame.data.byteLength - 2, 2); - const ivLength = frameTrailer[0]; - const iv = new Uint8Array( - encodedFrame.data, - encodedFrame.data.byteLength - ivLength - frameTrailer.byteLength, - ivLength, - ); - - const cipherTextLength = - encodedFrame.data.byteLength - - (frameHeader.byteLength + ivLength + frameTrailer.byteLength); + const frameInfo = this.extractFrameInfo(new Uint8Array(encodedFrame.data)); + const headerLength = this.getUnencryptedBytes(encodedFrame); const decryptedData = await this.decrypt( - new Uint8Array(encodedFrame.data, frameHeader.byteLength, cipherTextLength), - new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength), - iv, + frameInfo.encryptedPayload.slice(headerLength), + frameInfo.encryptedPayload.slice(0, headerLength), + frameInfo.iv, ratchetOpts?.encryptionKey ?? keySet!.encryptionKey, ); @@ -513,7 +477,6 @@ export class FrameCryptor extends BaseFrameCryptor { return encodedFrame; } catch (error: any) { - throw error; if (this.keyProviderOptions.ratchetWindowSize > 0) { if (ratchetOpts.ratchetCount < this.keyProviderOptions.ratchetWindowSize) { workerLogger.debug( @@ -562,7 +525,7 @@ export class FrameCryptor extends BaseFrameCryptor { } else { throw new CryptorError( `Decryption failed: ${error.message}`, - CryptorErrorReason.InvalidKey, + CryptorErrorReason.InternalError, ); } } @@ -632,9 +595,9 @@ export class FrameCryptor extends BaseFrameCryptor { } private getUnencryptedBytes(frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame): number { - if (isVideoFrame(frame)) { - let detectedCodec = this.getVideoCodec(frame) ?? this.videoCodec; + let detectedCodec = this.getCodec(frame) ?? this.codec; + if (isVideoFrame(frame)) { if (detectedCodec === 'av1' || detectedCodec === 'vp9') { throw new Error(`${detectedCodec} is not yet supported for end to end encryption`); } @@ -673,24 +636,42 @@ export class FrameCryptor extends BaseFrameCryptor { return UNENCRYPTED_BYTES[frame.type]; } else { - // @ts-expect-error payload type not supported on safari - if (frame.getMetadata().payloadType === 63) { + if (detectedCodec === 'red') { return UNENCRYPTED_BYTES.red1; } return UNENCRYPTED_BYTES.audio; } } + private extractFrameInfo(data: Uint8Array) { + const frameTrailer = data.slice(data.byteLength - 2); + const ivLength = frameTrailer[0]; + + const encryptedPayload = data.slice(0, data.byteLength - 2 - ivLength); + + const keyIndex = frameTrailer[1]; + + const iv = data.slice( + data.byteLength - ivLength - frameTrailer.byteLength, + data.byteLength - frameTrailer.byteLength, + ); + + return { encryptedPayload, keyIndex, iv }; + } + /** - * inspects frame payloadtype if available and maps it to the codec specified in rtpMap + * inspects frame payloadtype if available and maps it to the codec specified in rtpMap. + * falls back to this.codec */ - private getVideoCodec(frame: RTCEncodedVideoFrame): VideoCodec | undefined { + private getCodec( + frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, + ): VideoCodec | AudioCodec | undefined { if (this.rtpMap.size === 0) { return undefined; } // @ts-expect-error payloadType is not yet part of the typescript definition and currently not supported in Safari const payloadType = frame.getMetadata().payloadType; - const codec = payloadType ? this.rtpMap.get(payloadType) : undefined; + const codec = payloadType ? this.rtpMap.get(payloadType) : this.codec; return codec; } } diff --git a/src/e2ee/worker/e2ee.worker.ts b/src/e2ee/worker/e2ee.worker.ts index 5d3b7eccef..e3ca368f10 100644 --- a/src/e2ee/worker/e2ee.worker.ts +++ b/src/e2ee/worker/e2ee.worker.ts @@ -85,7 +85,7 @@ onmessage = (ev) => { unsetCryptorParticipant(data.trackId); break; case 'updateCodec': - getTrackCryptor(data.participantIdentity, data.trackId).setVideoCodec(data.codec); + getTrackCryptor(data.participantIdentity, data.trackId).setCodec(data.codec); break; case 'setRTPMap': // this is only used for the local participant diff --git a/src/room/PCTransport.ts b/src/room/PCTransport.ts index 62fa4fe5ed..1b999a8b4f 100644 --- a/src/room/PCTransport.ts +++ b/src/room/PCTransport.ts @@ -26,6 +26,7 @@ export const PCEvents = { NegotiationStarted: 'negotiationStarted', NegotiationComplete: 'negotiationComplete', RTPVideoPayloadTypes: 'rtpVideoPayloadTypes', + RTPAudioPayloadTypes: 'rtpAudioPayloadTypes', } as const; /** @internal */ @@ -150,6 +151,8 @@ export default class PCTransport extends EventEmitter { sdpParsed.media.forEach((media) => { if (media.type === 'video') { this.emit(PCEvents.RTPVideoPayloadTypes, media.rtp); + } else if (media.type === 'audio') { + this.emit(PCEvents.RTPAudioPayloadTypes, media.rtp); } }); } diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index 5272a5e353..15c9d95cc6 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -47,9 +47,10 @@ import type LocalTrack from './track/LocalTrack'; import type LocalVideoTrack from './track/LocalVideoTrack'; import type { SimulcastTrackInfo } from './track/LocalVideoTrack'; import { Track } from './track/Track'; -import type { TrackPublishOptions, VideoCodec } from './track/options'; +import type { AudioCodec, TrackPublishOptions, VideoCodec } from './track/options'; import { Mutex, + isAudioCodec, isVideoCodec, isWeb, sleep, @@ -1287,6 +1288,17 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit this.emit(EngineEvent.RTPVideoMapUpdate, rtpMap); }); + this.publisher.once(PCEvents.RTPAudioPayloadTypes, (rtpTypes: MediaAttributes['rtp']) => { + const rtpMap = new Map(); + rtpTypes.forEach((rtp) => { + const codec = rtp.codec.toLowerCase(); + if (isAudioCodec(codec)) { + rtpMap.set(rtp.payload, codec); + } + }); + this.emit(EngineEvent.RTPAudioMapUpdate, rtpMap); + }); + this.publisher.negotiate((e) => { cleanup(); reject(e); @@ -1411,6 +1423,7 @@ export type EngineEventCallbacks = { /** @internal */ trackSenderAdded: (track: Track, sender: RTCRtpSender) => void; rtpVideoMapUpdate: (rtpMap: Map) => void; + rtpAudioMapUpdate: (rtpMap: Map) => void; dcBufferStatusChanged: (isLow: boolean, kind: DataPacket_Kind) => void; participantUpdate: (infos: ParticipantInfo[]) => void; roomUpdate: (room: RoomModel) => void; diff --git a/src/room/events.ts b/src/room/events.ts index e136389ae2..92d40809f4 100644 --- a/src/room/events.ts +++ b/src/room/events.ts @@ -467,6 +467,7 @@ export enum EngineEvent { ActiveSpeakersUpdate = 'activeSpeakersUpdate', DataPacketReceived = 'dataPacketReceived', RTPVideoMapUpdate = 'rtpVideoMapUpdate', + RTPAudioMapUpdate = 'rtpAudioMapUpdate', DCBufferStatusChanged = 'dcBufferStatusChanged', ParticipantUpdate = 'participantUpdate', RoomUpdate = 'roomUpdate', diff --git a/src/room/track/options.ts b/src/room/track/options.ts index ce8585fd04..506da7b647 100644 --- a/src/room/track/options.ts +++ b/src/room/track/options.ts @@ -282,8 +282,12 @@ const backupCodecs = ['vp8', 'h264'] as const; export const videoCodecs = ['vp8', 'h264', 'vp9', 'av1'] as const; +export const audioCodecs = ['opus', 'red'] as const; + export type VideoCodec = (typeof videoCodecs)[number]; +export type AudioCodec = (typeof audioCodecs)[number]; + export type BackupVideoCodec = (typeof backupCodecs)[number]; export function isBackupCodec(codec: string): codec is BackupVideoCodec { diff --git a/src/room/utils.ts b/src/room/utils.ts index 8049b2a562..386de96b5b 100644 --- a/src/room/utils.ts +++ b/src/room/utils.ts @@ -4,7 +4,7 @@ import { getBrowser } from '../utils/browserParser'; import { protocolVersion, version } from '../version'; import type LocalAudioTrack from './track/LocalAudioTrack'; import type RemoteAudioTrack from './track/RemoteAudioTrack'; -import { VideoCodec, videoCodecs } from './track/options'; +import { AudioCodec, VideoCodec, audioCodecs, videoCodecs } from './track/options'; import { getNewAudioContext } from './track/utils'; import type { LiveKitReactNativeInfo } from './types'; @@ -463,6 +463,10 @@ export function isVideoCodec(maybeCodec: string): maybeCodec is VideoCodec { return videoCodecs.includes(maybeCodec as VideoCodec); } +export function isAudioCodec(maybeCodec: string): maybeCodec is AudioCodec { + return audioCodecs.includes(maybeCodec as AudioCodec); +} + export function unwrapConstraint(constraint: ConstrainDOMString): string { if (typeof constraint === 'string') { return constraint; From 78335c56c4191433be738b3915e2adf1a3ddb36a Mon Sep 17 00:00:00 2001 From: lukasIO Date: Tue, 5 Sep 2023 14:04:57 +0200 Subject: [PATCH 6/7] firefox split red working --- src/e2ee/E2eeManager.ts | 4 ++- src/e2ee/worker/FrameCryptor.ts | 22 +++++++++++++++-- src/e2ee/worker/e2ee.worker.ts | 12 ++++++--- src/room/PCTransport.ts | 25 ++++++++++--------- src/room/RTCEngine.ts | 44 +++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index 0b3d75aaed..fd4335856e 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -206,6 +206,8 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter { + console.log('sender info', publication.trackInfo); + this.setupE2EESender(publication.track!, publication.track!.sender!); }); @@ -389,7 +391,7 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter) { - this.rtpMap = map; + for (const [key, val] of map) { + this.rtpMap.set(key, val); + } } setupTransform( @@ -229,6 +232,7 @@ export class FrameCryptor extends BaseFrameCryptor { try { if (this.getCodec(encodedFrame) === 'red') { const { primaryBlock, redundancyBlocks } = red.split(encodedFrame.data); + console.log({ primaryBlock, redundancyBlocks }); const primaryBlockEncrypted = await this.encrypt( primaryBlock.data.slice(1), @@ -411,7 +415,11 @@ export class FrameCryptor extends BaseFrameCryptor { // ---------+-------------------------+-+---------+---- try { + if (encodedFrame instanceof RTCEncodedAudioFrame) { + console.log('audio info', encodedFrame.getMetadata()); + } if (this.getCodec(encodedFrame) === 'red') { + console.log('using red'); const { primaryBlock, redundancyBlocks } = red.split(encodedFrame.data); const primaryFrame = this.extractFrameInfo(primaryBlock.data); @@ -466,6 +474,10 @@ export class FrameCryptor extends BaseFrameCryptor { const frameInfo = this.extractFrameInfo(new Uint8Array(encodedFrame.data)); const headerLength = this.getUnencryptedBytes(encodedFrame); + if (encodedFrame instanceof RTCEncodedAudioFrame) { + console.log('audio info', frameInfo, encodedFrame.getMetadata()); + } + const decryptedData = await this.decrypt( frameInfo.encryptedPayload.slice(headerLength), frameInfo.encryptedPayload.slice(0, headerLength), @@ -477,6 +489,7 @@ export class FrameCryptor extends BaseFrameCryptor { return encodedFrame; } catch (error: any) { + throw error; if (this.keyProviderOptions.ratchetWindowSize > 0) { if (ratchetOpts.ratchetCount < this.keyProviderOptions.ratchetWindowSize) { workerLogger.debug( @@ -666,9 +679,14 @@ export class FrameCryptor extends BaseFrameCryptor { private getCodec( frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, ): VideoCodec | AudioCodec | undefined { + // console.log('codec info', this.codec, this.rtpMap); + // if (isFireFox() || frame instanceof RTCEncodedAudioFrame) { + // return 'opus'; + // } if (this.rtpMap.size === 0) { - return undefined; + return this.codec; } + // @ts-expect-error payloadType is not yet part of the typescript definition and currently not supported in Safari const payloadType = frame.getMetadata().payloadType; const codec = payloadType ? this.rtpMap.get(payloadType) : this.codec; diff --git a/src/e2ee/worker/e2ee.worker.ts b/src/e2ee/worker/e2ee.worker.ts index e3ca368f10..d79bfa7ce7 100644 --- a/src/e2ee/worker/e2ee.worker.ts +++ b/src/e2ee/worker/e2ee.worker.ts @@ -1,4 +1,5 @@ import { workerLogger } from '../../logger'; +import type { AudioCodec, VideoCodec } from '../../room/track/options'; import { KEY_PROVIDER_DEFAULTS } from '../constants'; import { CryptorErrorReason } from '../errors'; import { CryptorEvent, KeyHandlerEvent } from '../events'; @@ -27,6 +28,8 @@ let sifTrailer: Uint8Array | undefined; let keyProviderOptions: KeyProviderOptions = KEY_PROVIDER_DEFAULTS; +const rtpMap = new Map(); + workerLogger.setDefaultLevel('info'); onmessage = (ev) => { @@ -89,10 +92,11 @@ onmessage = (ev) => { break; case 'setRTPMap': // this is only used for the local participant + for (const [key, val] of data.map) { + rtpMap.set(key, val); + } participantCryptors.forEach((cr) => { - if (cr.getParticipantIdentity() === data.participantIdentity) { - cr.setRtpMap(data.map); - } + cr.setRtpMap(rtpMap); }); break; case 'ratchetRequest': @@ -135,8 +139,8 @@ function getTrackCryptor(participantIdentity: string, trackId: string) { keyProviderOptions, sifTrailer, }); - setupCryptorErrorEvents(cryptor); + cryptor.setRtpMap(rtpMap); participantCryptors.push(cryptor); } else if (participantIdentity !== cryptor.getParticipantIdentity()) { // assign new participant id to track cryptor and pass in correct key handler diff --git a/src/room/PCTransport.ts b/src/room/PCTransport.ts index 1b999a8b4f..05ba53be35 100644 --- a/src/room/PCTransport.ts +++ b/src/room/PCTransport.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'events'; -import type { MediaDescription } from 'sdp-transform'; +import type { MediaDescription, SessionDescription } from 'sdp-transform'; import { parse, write } from 'sdp-transform'; import { debounce } from 'ts-debounce'; import log from '../logger'; @@ -146,16 +146,6 @@ export default class PCTransport extends EventEmitter { this.createAndSendOffer(); } else if (sd.type === 'answer') { this.emit(PCEvents.NegotiationComplete); - if (sd.sdp) { - const sdpParsed = parse(sd.sdp); - sdpParsed.media.forEach((media) => { - if (media.type === 'video') { - this.emit(PCEvents.RTPVideoPayloadTypes, media.rtp); - } else if (media.type === 'audio') { - this.emit(PCEvents.RTPAudioPayloadTypes, media.rtp); - } - }); - } } } @@ -258,6 +248,7 @@ export default class PCTransport extends EventEmitter { }); await this.setMungedSDP(offer, write(sdpParsed)); + this.emitRTPMaps(sdpParsed); this.onOffer(offer); } @@ -270,6 +261,8 @@ export default class PCTransport extends EventEmitter { } }); await this.setMungedSDP(answer, write(sdpParsed)); + this.emitRTPMaps(sdpParsed); + return answer; } @@ -344,6 +337,16 @@ export default class PCTransport extends EventEmitter { throw new NegotiationError(msg); } } + + emitRTPMaps(sdpParsed: SessionDescription) { + sdpParsed.media.forEach((media) => { + if (media.type === 'video') { + this.emit(PCEvents.RTPVideoPayloadTypes, media.rtp); + } else if (media.type === 'audio') { + this.emit(PCEvents.RTPAudioPayloadTypes, media.rtp); + } + }); + } } function ensureAudioNackAndStereo( diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index 15c9d95cc6..997e587cf7 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -452,6 +452,50 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit this.emit(EngineEvent.MediaTrackAdded, ev.track, ev.streams[0], ev.receiver); }; + this.publisher.once(PCEvents.RTPVideoPayloadTypes, (rtpTypes: MediaAttributes['rtp']) => { + const rtpMap = new Map(); + rtpTypes.forEach((rtp) => { + const codec = rtp.codec.toLowerCase(); + if (isVideoCodec(codec)) { + rtpMap.set(rtp.payload, codec); + } + }); + this.emit(EngineEvent.RTPVideoMapUpdate, rtpMap); + }); + + this.publisher.once(PCEvents.RTPAudioPayloadTypes, (rtpTypes: MediaAttributes['rtp']) => { + const rtpMap = new Map(); + rtpTypes.forEach((rtp) => { + const codec = rtp.codec.toLowerCase(); + if (isAudioCodec(codec)) { + rtpMap.set(rtp.payload, codec); + } + }); + this.emit(EngineEvent.RTPAudioMapUpdate, rtpMap); + }); + + this.subscriber.once(PCEvents.RTPVideoPayloadTypes, (rtpTypes: MediaAttributes['rtp']) => { + const rtpMap = new Map(); + rtpTypes.forEach((rtp) => { + const codec = rtp.codec.toLowerCase(); + if (isVideoCodec(codec)) { + rtpMap.set(rtp.payload, codec); + } + }); + this.emit(EngineEvent.RTPVideoMapUpdate, rtpMap); + }); + + this.subscriber.once(PCEvents.RTPAudioPayloadTypes, (rtpTypes: MediaAttributes['rtp']) => { + const rtpMap = new Map(); + rtpTypes.forEach((rtp) => { + const codec = rtp.codec.toLowerCase(); + if (isAudioCodec(codec)) { + rtpMap.set(rtp.payload, codec); + } + }); + this.emit(EngineEvent.RTPAudioMapUpdate, rtpMap); + }); + this.createDataChannels(); } From 31d18e202e08a12e70801950beb759d010662396 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Tue, 5 Sep 2023 14:09:23 +0200 Subject: [PATCH 7/7] fix TS errors --- src/e2ee/worker/FrameCryptor.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index 52af16bfe2..c3145aa0c6 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -5,7 +5,6 @@ import * as red from 'opus-red-parser'; import type TypedEventEmitter from 'typed-emitter'; import { workerLogger } from '../../logger'; import type { AudioCodec, VideoCodec } from '../../room/track/options'; -import { isFireFox } from '../../room/utils'; import { ENCRYPTION_ALGORITHM, IV_LENGTH, UNENCRYPTED_BYTES } from '../constants'; import { CryptorError, CryptorErrorReason } from '../errors'; import { CryptorCallbacks, CryptorEvent } from '../events'; @@ -489,7 +488,6 @@ export class FrameCryptor extends BaseFrameCryptor { return encodedFrame; } catch (error: any) { - throw error; if (this.keyProviderOptions.ratchetWindowSize > 0) { if (ratchetOpts.ratchetCount < this.keyProviderOptions.ratchetWindowSize) { workerLogger.debug(