diff --git a/app/images/audio.svg b/app/images/audio.svg new file mode 100644 index 000000000..6f2c1831f --- /dev/null +++ b/app/images/audio.svg @@ -0,0 +1,92 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + diff --git a/app/ui.js b/app/ui.js index f27dfe28e..64b861f3d 100644 --- a/app/ui.js +++ b/app/ui.js @@ -232,6 +232,9 @@ const UI = { document.getElementById("noVNC_view_drag_button") .addEventListener('click', UI.toggleViewDrag); + document.getElementById("noVNC_audio_button") + .addEventListener('click', UI.toggleEnableAudio); + document.getElementById("noVNC_control_bar_handle") .addEventListener('mousedown', UI.controlbarHandleMouseDown); document.getElementById("noVNC_control_bar_handle") @@ -448,7 +451,7 @@ const UI = { UI.enableSetting('port'); UI.enableSetting('path'); UI.enableSetting('repeaterID'); - UI.updatePowerButton(); + UI.updateCapabilities(); UI.keepControlbar(); } @@ -891,6 +894,24 @@ const UI = { } }, + updateCapabilities() { + UI.updatePowerButton(); + UI.updateAudioButton(); + }, + + updateAudioButton() { + if (UI.connected && + UI.rfb.capabilities.audio) { + document.getElementById('noVNC_audio_button') + .classList.remove("noVNC_hidden"); + document.getElementById('noVNC_audio_button') + .classList.remove("noVNC_selected"); + } else { + document.getElementById('noVNC_audio_button') + .classList.add("noVNC_hidden"); + } + }, + /* ------^------- * /SETTINGS * ============== @@ -1059,7 +1080,7 @@ const UI = { UI.rfb.addEventListener("credentialsrequired", UI.credentials); UI.rfb.addEventListener("securityfailure", UI.securityFailed); UI.rfb.addEventListener("clippingviewport", UI.updateViewDrag); - UI.rfb.addEventListener("capabilities", UI.updatePowerButton); + UI.rfb.addEventListener("capabilities", UI.updateCapabilities); UI.rfb.addEventListener("clipboard", UI.clipboardReceive); UI.rfb.addEventListener("bell", UI.bell); UI.rfb.addEventListener("desktopname", UI.updateDesktopName); @@ -1722,6 +1743,27 @@ const UI = { } }, + toggleEnableAudio() { + if (!UI.rfb) return; + + if (!document.getElementById('noVNC_audio_button') + .classList.contains("noVNC_selected")) { + UI.rfb.enableAudio( + 2, + MediaSource.isTypeSupported('audio/webm;codecs=opus') ? + RFB.audioCodecs.OpusWebM : + RFB.audioCodecs.MP3, + 32 * 1024 // 32kbps + ); + document.getElementById('noVNC_audio_button') + .classList.add("noVNC_selected"); + } else { + UI.rfb.disableAudio(); + document.getElementById('noVNC_audio_button') + .classList.remove("noVNC_selected"); + } + }, + updateShowDotCursor() { if (!UI.rfb) return; UI.rfb.showDotCursor = UI.getSetting('show_dot'); diff --git a/core/encodings.js b/core/encodings.js index 1a79989d1..5a130f623 100644 --- a/core/encodings.js +++ b/core/encodings.js @@ -30,6 +30,7 @@ export const encodings = { pseudoEncodingContinuousUpdates: -313, pseudoEncodingCompressLevel9: -247, pseudoEncodingCompressLevel0: -256, + pseudoEncodingReplitAudio: 0x52706c41, pseudoEncodingVMwareCursor: 0x574d5664, pseudoEncodingExtendedClipboard: 0xc0a1e5ce }; diff --git a/core/rfb.js b/core/rfb.js index f2deb0e7b..b408afd47 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -13,6 +13,7 @@ import { encodeUTF8, decodeUTF8 } from './util/strings.js'; import { dragThreshold } from './util/browser.js'; import { clientToElement } from './util/element.js'; import { setCapture } from './util/events.js'; +import AudioStream from './util/audio.js'; import EventTargetMixin from './util/eventtarget.js'; import Display from "./display.js"; import Inflator from "./inflator.js"; @@ -137,7 +138,7 @@ export default class RFB extends EventTargetMixin { this._fbName = ""; - this._capabilities = { power: false }; + this._capabilities = { power: false, audio: false }; this._supportsFence = false; @@ -149,6 +150,8 @@ export default class RFB extends EventTargetMixin { this._screenFlags = 0; this._qemuExtKeyEventSupported = false; + this._replitAudioSupported = false; + this._replitAudioServerVersion = -1; this._clipboardText = null; this._clipboardServerCapabilitiesActions = {}; @@ -195,6 +198,11 @@ export default class RFB extends EventTargetMixin { this._gestureLastMagnitudeX = 0; this._gestureLastMagnitudeY = 0; + // Audio state + this._audioEnabled = false; + this._audioMimeType = null; + this._audioStream = null; + // Bound event handlers this._eventHandlers = { focusCanvas: this._focusCanvas.bind(this), @@ -541,6 +549,25 @@ export default class RFB extends EventTargetMixin { return this._display.toBlob(callback, type, quality); } + enableAudio(channels, codec, kbps) { + if (this._audioEnabled) { return; } + + this._audioEnabled = true; + if (codec == RFB.audioCodecs.OpusWebM) { + this._audioMimeType = 'audio/webm;codecs=opus'; + } else if (codec == RFB.audioCodecs.MP3) { + this._audioMimeType = 'audio/mpeg'; + } + RFB.messages.ReplitAudioStartEncoder(this._sock, true, channels, codec, kbps); + } + + disableAudio() { + if (!this._audioEnabled) { return; } + + this._audioEnabled = false; + RFB.messages.ReplitAudioStartEncoder(this._sock, false, 0, 0, 0); + } + // ===== PRIVATE METHODS ===== _connect() { @@ -2132,6 +2159,7 @@ export default class RFB extends EventTargetMixin { encs.push(encodings.pseudoEncodingLastRect); encs.push(encodings.pseudoEncodingQEMUExtendedKeyEvent); encs.push(encodings.pseudoEncodingQEMULedEvent); + encs.push(encodings.pseudoEncodingReplitAudio); encs.push(encodings.pseudoEncodingExtendedDesktopSize); encs.push(encodings.pseudoEncodingXvp); encs.push(encodings.pseudoEncodingFence); @@ -2411,6 +2439,54 @@ export default class RFB extends EventTargetMixin { return true; } + _handleReplitAudioPseudoEncodingMsg() { + if (this._sock.rQwait("Repl.it audio message", 3, 1)) { return false; } + const submessage = this._sock.rQshift8(); + const length = this._sock.rQshift16(); + if (this._sock.rQwait("Repl.it audio message", length, 4)) { return false; } + + switch (submessage) { + case 0: { // StartCapture response. + const enabled = this._sock.rQshift8() == 1; + + if (enabled) { + this._audioStream = new AudioStream(this._audioMimeType); + RFB.messages.ReplitAudioEnableContinuousUpdate(this._sock); + } else if (this._audioStream != null) { + this._audioStream.close(); + this._audioStream = null; + } + break; + } + + case 1: { // AudioFrame response. + const keyframeAndTimestamp = this._sock.rQshift32(); + const keyframe = (keyframeAndTimestamp & 0x80000000) != 0; + const timestamp = keyframeAndTimestamp & 0x7fffffff; + const data = this._sock.rQshiftBytes(length - 4); + if (this._audioStream != null) { + this._audioStream.queueAudioFrame(timestamp / 1000, keyframe, data); + } + break; + } + + case 2: { // StartContinuousUpdates response. + const enabled = this._sock.rQshift8() == 1; + if (!enabled && this._audioStream != null) { + this._audioStream.close(); + this._audioStream = null; + } + break; + } + + default: + this._fail("Illegal server Repl.it audio message (msg: " + submessage + ")"); + break; + } + + return true; + } + _handleXvpMsg() { if (this._sock.rQwait("XVP version and message", 3, 1)) { return false; } this._sock.rQskipBytes(1); // Padding @@ -2479,6 +2555,9 @@ export default class RFB extends EventTargetMixin { } return true; + case 245: // Repl.it audio message + return this._handleReplitAudioPseudoEncodingMsg(); + case 248: // ServerFence return this._handleServerFenceMsg(); @@ -2557,6 +2636,9 @@ export default class RFB extends EventTargetMixin { this._qemuExtKeyEventSupported = true; return true; + case encodings.pseudoEncodingReplitAudio: + return this._handleReplitAudioPseudoEncoding(); + case encodings.pseudoEncodingDesktopName: return this._handleDesktopName(); @@ -2728,6 +2810,25 @@ export default class RFB extends EventTargetMixin { return true; } + _handleReplitAudioPseudoEncoding() { + if (this._sock.rQwait("Repl.it audio", 4)) { + return false; + } + + const version = this._sock.rQshift16(); + const codecs = this._sock.rQshift16(); + + if (this._sock.rQwait("Repl.it audio", 2 * codecs, 4)) { + return false; + } + this._sock.rQshiftStr(2 * codecs); + + this._replitAudioSupported = true; + this._replitAudioServerVersion = version; + this._setCapability("audio", true); + return true; + } + _handleDesktopName() { if (this._sock.rQwait("DesktopName", 4)) { return false; @@ -2935,6 +3036,12 @@ export default class RFB extends EventTargetMixin { } } +// Audio codecs +RFB.audioCodecs = { + OpusWebM: 0, + MP3: 1, +}; + // Class Methods RFB.messages = { keyEvent(sock, keysym, down) { @@ -2948,6 +3055,54 @@ RFB.messages = { sock.flush(); }, + ReplitAudioStartEncoder(sock, enabled, channels, codec, kbps) { + const buff = sock._sQ; + const offset = sock._sQlen; + + buff[offset] = 245; // msg-type + buff[offset + 1] = 0; // sub msg-type + buff[offset + 2] = 0; + buff[offset + 3] = 6; // length + + buff[offset + 4] = enabled ? 1 : 0; // enabled + buff[offset + 5] = channels; + + buff[offset + 6] = codec >> 8; + buff[offset + 7] = codec; + + buff[offset + 8] = kbps >> 8; + buff[offset + 9] = kbps; + + sock._sQlen += 10; + sock.flush(); + }, + + ReplitAudioRequestFrame(sock, channels, codec, kbps) { + const buff = sock._sQ; + const offset = sock._sQlen; + + buff[offset] = 245; // msg-type + buff[offset + 1] = 1; // sub msg-type + buff[offset + 2] = 0; + buff[offset + 3] = 0; // length + + sock._sQlen += 4; + sock.flush(); + }, + + ReplitAudioEnableContinuousUpdate(sock) { + const buff = sock._sQ; + const offset = sock._sQlen; + + buff[offset] = 245; // msg-type + buff[offset + 1] = 2; // sub msg-type + buff[offset + 2] = 0; + buff[offset + 3] = 0; // length + + sock._sQlen += 4; + sock.flush(); + }, + QEMUExtendedKeyEvent(sock, keysym, down, keycode) { function getRFBkeycode(xtScanCode) { const upperByte = (keycode >> 8); diff --git a/core/util/audio.js b/core/util/audio.js new file mode 100644 index 000000000..dbfc40034 --- /dev/null +++ b/core/util/audio.js @@ -0,0 +1,201 @@ +/* + * noVNC: HTML5 VNC client + * Copyright (C) 2021 The noVNC Authors + * Licensed under MPL 2.0 (see LICENSE.txt) + * + * See README.md for usage and integration instructions. + */ + +// The maximum allowable de-sync, in seconds. If the time between the last +// received timestamp and the current audio playback timestamp exceeds this +// value, the audio stream will be seeked to the most current timestamp +// possible. +const MAX_ALLOWABLE_DESYNC = 0.5; + +// The amount of time, in seconds, to keep in the audio buffer while seeking. +// Whenever a de-sync event happens and we need to seek to a future +// timestamp, we skip to the last buffered time minus this amount, so that the +// browser has this amount of time worth of buffered audio data. This is done +// to avoid having the browser enter a buffering state just after seeking. +const SEEK_BUFFER_LENGTH = 0.2; + +// An audio stream built upon Media Stream Extensions. +export default class AudioStream { + constructor(codec) { + this._codec = codec; + this._reset(); + } + + _reset() { + // Instantiate a media source and audio buffer/queue. + this._mediaSource = new MediaSource(); + this._audioBuffer = null; + this._audioQ = []; + + // Create a hidden audio element. + this._audio = document.createElement("audio"); + this._audio.src = window.URL.createObjectURL(this._mediaSource); + + // When data is queued, start playing. + this._audio.autoplay = true; + this._mediaSource.addEventListener( + "sourceopen", + this._onSourceOpen.bind(this), + false + ); + this._audio.addEventListener( + "error", + (ev) => { + console.error("Audio element error", ev); + }, + false + ); + this._audio.addEventListener("canplay", () => { + try { + this._audio.play(); + } catch (e) { + // Firefox and Chrome are totally cool with playing this + // the moment we can do it, but Safari throws an exception + // since play() is not called in a stack that ran a user + // event handler. + } + }); + } + + _onSourceOpen(e) { + if (this._audioBuffer) { + return; + } + this._audioBuffer = this._mediaSource.addSourceBuffer(this._codec); + this._audioBuffer.mode = "segments"; + this._audioBuffer.addEventListener( + "updateend", + this._onUpdateBuffer.bind(this) + ); + this._audioBuffer.addEventListener("error", (ev) => { + console.error("AudioBuffer error", ev); + }); + } + + _onUpdateBuffer() { + if ( + !this._audioBuffer || + this._audioBuffer.updating || + this._audio.error + ) { + // The audio buffer is not yet ready to accept any new data. + return; + } + if (!this._audioQ.length) { + // There's nothing to append. + return; + } + + const timestamp = this._audioQ[0][0]; + if (this._audioQ.length === 1) { + this._appendChunk(timestamp, this._audioQ.pop()[1]); + return; + } + + // If there is more than one chunk in the queue, they are coalesced + // into a single buffer. This is because following appendBuffer(), + // the audio buffer changes to an "updating" state for a small amount + // of time and any new chunks won't be able to be appended immediately. + // Since the internal queue is used when the browser is trying to catch + // up with the server, we want to have the audio buffer unappendable + // for a smaller amount of time. + let chunkLength = 0; + for (let i = 0; i < this._audioQ.length; ++i) { + chunkLength += this._audioQ[i][1].byteLength; + } + const chunk = new Uint8Array(chunkLength); + let offset = 0; + for (let i = 0; i < this._audioQ.length; ++i) { + chunk.set(new Uint8Array(this._audioQ[i][1]), offset); + offset += this._audioQ[i][1].byteLength; + } + this._audioQ.splice(0, this._audioQ.length); + this._appendChunk(timestamp, chunk); + } + + // Append a chunk into the AudioBuffer. The caller should ensure that + // the AudioBuffer is ready to receive the chunk. If the difference + // between the current playback position of the audio and the timestamp + // exceeds the maximum allowable desync threshold, the audio will be + // seeked to the latest possible position that doesn't trigger buffering + // to avoid an arbitrarily large desync between video and audio. + _appendChunk(timestamp, chunk) { + this._audioBuffer.appendBuffer(chunk); + if ( + timestamp - this._audio.currentTime > MAX_ALLOWABLE_DESYNC && + (this._audio.seekable.length || this._audio.buffered.length) + ) { + console.debug("maximum allowable desync reached", { + readyState: this._audio.readyState, + buffered: ( + (this._audio.buffered && + this._audio.buffered.length && + this._audio.buffered.end( + this._audio.buffered.length - 1 + )) || + 0 + ).toFixed(2), + seekable: ( + (this._audio.seekable && + this._audio.seekable.length && + this._audio.seekable.end( + this._audio.seekable.length - 1 + )) || + 0 + ).toFixed(2), + time: this._audio.currentTime.toFixed(2), + delta: (timestamp - this._audio.currentTime).toFixed(2) + }); + if (this._audio.buffered && this._audio.buffered.length) { + this._audio.currentTime = + this._audio.buffered.end(this._audio.buffered.length - 1) - + SEEK_BUFFER_LENGTH; + } else { + this._audio.currentTime = + this._audio.seekable.end(this._audio.seekable.length - 1) - + SEEK_BUFFER_LENGTH; + } + } + } + + // Queues an audio chunk at a particular timestamp. + queueAudioFrame(timestamp, keyframe, chunk) { + // If the MSE audio buffer is not ready to receive the chunk or + // there are some other chunks waiting to be appended, we save + // a copy of it into our own internal queue. Eventually, + // when it becomes ready, we append all pending chunks at once. + if ( + this._audioBuffer === null || + this._audioBuffer.updating || + this._audio.error || + this._audioQ.length + ) { + // We need to make a copy, since `chunk` is a view of the underlying + // buffer owned by Websock, and will be mutated once we return. + // TODO: `keyframe` can be used to decide when to drop a chunk if + // there's enough backpressure. + const copy = new ArrayBuffer(chunk.byteLength); + new Uint8Array(copy).set(new Uint8Array(chunk)); + this._audioQ.push([timestamp, copy]); + this._onUpdateBuffer(); + return; + } + + this._appendChunk(timestamp, chunk); + } + + close() { + if (this._audio) { + this._audio.pause(); + } + this._mediaSource = null; + this._audioBuffer = null; + this._audioQ = []; + this._audio = null; + } +} diff --git a/vnc.html b/vnc.html index 24a118dbd..a7b79d1a9 100644 --- a/vnc.html +++ b/vnc.html @@ -108,6 +108,11 @@

no
VNC

+ + +