diff --git a/app/images/audio.svg b/app/images/audio.svg
new file mode 100644
index 00000000..6f2c1831
--- /dev/null
+++ b/app/images/audio.svg
@@ -0,0 +1,92 @@
+
+
diff --git a/app/ui.js b/app/ui.js
index f27dfe28..64b861f3 100644
--- a/app/ui.js
+++ b/app/ui.js
@@ -232,6 +232,9 @@ const UI = {
document.getElementById("noVNC_view_drag_button")
.addEventListener('click', UI.toggleViewDrag);
+ document.getElementById("noVNC_audio_button")
+ .addEventListener('click', UI.toggleEnableAudio);
+
document.getElementById("noVNC_control_bar_handle")
.addEventListener('mousedown', UI.controlbarHandleMouseDown);
document.getElementById("noVNC_control_bar_handle")
@@ -448,7 +451,7 @@ const UI = {
UI.enableSetting('port');
UI.enableSetting('path');
UI.enableSetting('repeaterID');
- UI.updatePowerButton();
+ UI.updateCapabilities();
UI.keepControlbar();
}
@@ -891,6 +894,24 @@ const UI = {
}
},
+ updateCapabilities() {
+ UI.updatePowerButton();
+ UI.updateAudioButton();
+ },
+
+ updateAudioButton() {
+ if (UI.connected &&
+ UI.rfb.capabilities.audio) {
+ document.getElementById('noVNC_audio_button')
+ .classList.remove("noVNC_hidden");
+ document.getElementById('noVNC_audio_button')
+ .classList.remove("noVNC_selected");
+ } else {
+ document.getElementById('noVNC_audio_button')
+ .classList.add("noVNC_hidden");
+ }
+ },
+
/* ------^-------
* /SETTINGS
* ==============
@@ -1059,7 +1080,7 @@ const UI = {
UI.rfb.addEventListener("credentialsrequired", UI.credentials);
UI.rfb.addEventListener("securityfailure", UI.securityFailed);
UI.rfb.addEventListener("clippingviewport", UI.updateViewDrag);
- UI.rfb.addEventListener("capabilities", UI.updatePowerButton);
+ UI.rfb.addEventListener("capabilities", UI.updateCapabilities);
UI.rfb.addEventListener("clipboard", UI.clipboardReceive);
UI.rfb.addEventListener("bell", UI.bell);
UI.rfb.addEventListener("desktopname", UI.updateDesktopName);
@@ -1722,6 +1743,27 @@ const UI = {
}
},
+ toggleEnableAudio() {
+ if (!UI.rfb) return;
+
+ if (!document.getElementById('noVNC_audio_button')
+ .classList.contains("noVNC_selected")) {
+ UI.rfb.enableAudio(
+ 2,
+ MediaSource.isTypeSupported('audio/webm;codecs=opus') ?
+ RFB.audioCodecs.OpusWebM :
+ RFB.audioCodecs.MP3,
+ 32 * 1024 // 32kbps
+ );
+ document.getElementById('noVNC_audio_button')
+ .classList.add("noVNC_selected");
+ } else {
+ UI.rfb.disableAudio();
+ document.getElementById('noVNC_audio_button')
+ .classList.remove("noVNC_selected");
+ }
+ },
+
updateShowDotCursor() {
if (!UI.rfb) return;
UI.rfb.showDotCursor = UI.getSetting('show_dot');
diff --git a/core/encodings.js b/core/encodings.js
index 1a79989d..5a130f62 100644
--- a/core/encodings.js
+++ b/core/encodings.js
@@ -30,6 +30,7 @@ export const encodings = {
pseudoEncodingContinuousUpdates: -313,
pseudoEncodingCompressLevel9: -247,
pseudoEncodingCompressLevel0: -256,
+ pseudoEncodingReplitAudio: 0x52706c41,
pseudoEncodingVMwareCursor: 0x574d5664,
pseudoEncodingExtendedClipboard: 0xc0a1e5ce
};
diff --git a/core/rfb.js b/core/rfb.js
index f2deb0e7..b408afd4 100644
--- a/core/rfb.js
+++ b/core/rfb.js
@@ -13,6 +13,7 @@ import { encodeUTF8, decodeUTF8 } from './util/strings.js';
import { dragThreshold } from './util/browser.js';
import { clientToElement } from './util/element.js';
import { setCapture } from './util/events.js';
+import AudioStream from './util/audio.js';
import EventTargetMixin from './util/eventtarget.js';
import Display from "./display.js";
import Inflator from "./inflator.js";
@@ -137,7 +138,7 @@ export default class RFB extends EventTargetMixin {
this._fbName = "";
- this._capabilities = { power: false };
+ this._capabilities = { power: false, audio: false };
this._supportsFence = false;
@@ -149,6 +150,8 @@ export default class RFB extends EventTargetMixin {
this._screenFlags = 0;
this._qemuExtKeyEventSupported = false;
+ this._replitAudioSupported = false;
+ this._replitAudioServerVersion = -1;
this._clipboardText = null;
this._clipboardServerCapabilitiesActions = {};
@@ -195,6 +198,11 @@ export default class RFB extends EventTargetMixin {
this._gestureLastMagnitudeX = 0;
this._gestureLastMagnitudeY = 0;
+ // Audio state
+ this._audioEnabled = false;
+ this._audioMimeType = null;
+ this._audioStream = null;
+
// Bound event handlers
this._eventHandlers = {
focusCanvas: this._focusCanvas.bind(this),
@@ -541,6 +549,25 @@ export default class RFB extends EventTargetMixin {
return this._display.toBlob(callback, type, quality);
}
+ enableAudio(channels, codec, kbps) {
+ if (this._audioEnabled) { return; }
+
+ this._audioEnabled = true;
+ if (codec == RFB.audioCodecs.OpusWebM) {
+ this._audioMimeType = 'audio/webm;codecs=opus';
+ } else if (codec == RFB.audioCodecs.MP3) {
+ this._audioMimeType = 'audio/mpeg';
+ }
+ RFB.messages.ReplitAudioStartEncoder(this._sock, true, channels, codec, kbps);
+ }
+
+ disableAudio() {
+ if (!this._audioEnabled) { return; }
+
+ this._audioEnabled = false;
+ RFB.messages.ReplitAudioStartEncoder(this._sock, false, 0, 0, 0);
+ }
+
// ===== PRIVATE METHODS =====
_connect() {
@@ -2132,6 +2159,7 @@ export default class RFB extends EventTargetMixin {
encs.push(encodings.pseudoEncodingLastRect);
encs.push(encodings.pseudoEncodingQEMUExtendedKeyEvent);
encs.push(encodings.pseudoEncodingQEMULedEvent);
+ encs.push(encodings.pseudoEncodingReplitAudio);
encs.push(encodings.pseudoEncodingExtendedDesktopSize);
encs.push(encodings.pseudoEncodingXvp);
encs.push(encodings.pseudoEncodingFence);
@@ -2411,6 +2439,54 @@ export default class RFB extends EventTargetMixin {
return true;
}
+ _handleReplitAudioPseudoEncodingMsg() {
+ if (this._sock.rQwait("Repl.it audio message", 3, 1)) { return false; }
+ const submessage = this._sock.rQshift8();
+ const length = this._sock.rQshift16();
+ if (this._sock.rQwait("Repl.it audio message", length, 4)) { return false; }
+
+ switch (submessage) {
+ case 0: { // StartCapture response.
+ const enabled = this._sock.rQshift8() == 1;
+
+ if (enabled) {
+ this._audioStream = new AudioStream(this._audioMimeType);
+ RFB.messages.ReplitAudioEnableContinuousUpdate(this._sock);
+ } else if (this._audioStream != null) {
+ this._audioStream.close();
+ this._audioStream = null;
+ }
+ break;
+ }
+
+ case 1: { // AudioFrame response.
+ const keyframeAndTimestamp = this._sock.rQshift32();
+ const keyframe = (keyframeAndTimestamp & 0x80000000) != 0;
+ const timestamp = keyframeAndTimestamp & 0x7fffffff;
+ const data = this._sock.rQshiftBytes(length - 4);
+ if (this._audioStream != null) {
+ this._audioStream.queueAudioFrame(timestamp / 1000, keyframe, data);
+ }
+ break;
+ }
+
+ case 2: { // StartContinuousUpdates response.
+ const enabled = this._sock.rQshift8() == 1;
+ if (!enabled && this._audioStream != null) {
+ this._audioStream.close();
+ this._audioStream = null;
+ }
+ break;
+ }
+
+ default:
+ this._fail("Illegal server Repl.it audio message (msg: " + submessage + ")");
+ break;
+ }
+
+ return true;
+ }
+
_handleXvpMsg() {
if (this._sock.rQwait("XVP version and message", 3, 1)) { return false; }
this._sock.rQskipBytes(1); // Padding
@@ -2479,6 +2555,9 @@ export default class RFB extends EventTargetMixin {
}
return true;
+ case 245: // Repl.it audio message
+ return this._handleReplitAudioPseudoEncodingMsg();
+
case 248: // ServerFence
return this._handleServerFenceMsg();
@@ -2557,6 +2636,9 @@ export default class RFB extends EventTargetMixin {
this._qemuExtKeyEventSupported = true;
return true;
+ case encodings.pseudoEncodingReplitAudio:
+ return this._handleReplitAudioPseudoEncoding();
+
case encodings.pseudoEncodingDesktopName:
return this._handleDesktopName();
@@ -2728,6 +2810,25 @@ export default class RFB extends EventTargetMixin {
return true;
}
+ _handleReplitAudioPseudoEncoding() {
+ if (this._sock.rQwait("Repl.it audio", 4)) {
+ return false;
+ }
+
+ const version = this._sock.rQshift16();
+ const codecs = this._sock.rQshift16();
+
+ if (this._sock.rQwait("Repl.it audio", 2 * codecs, 4)) {
+ return false;
+ }
+ this._sock.rQshiftStr(2 * codecs);
+
+ this._replitAudioSupported = true;
+ this._replitAudioServerVersion = version;
+ this._setCapability("audio", true);
+ return true;
+ }
+
_handleDesktopName() {
if (this._sock.rQwait("DesktopName", 4)) {
return false;
@@ -2935,6 +3036,12 @@ export default class RFB extends EventTargetMixin {
}
}
+// Audio codecs
+RFB.audioCodecs = {
+ OpusWebM: 0,
+ MP3: 1,
+};
+
// Class Methods
RFB.messages = {
keyEvent(sock, keysym, down) {
@@ -2948,6 +3055,54 @@ RFB.messages = {
sock.flush();
},
+ ReplitAudioStartEncoder(sock, enabled, channels, codec, kbps) {
+ const buff = sock._sQ;
+ const offset = sock._sQlen;
+
+ buff[offset] = 245; // msg-type
+ buff[offset + 1] = 0; // sub msg-type
+ buff[offset + 2] = 0;
+ buff[offset + 3] = 6; // length
+
+ buff[offset + 4] = enabled ? 1 : 0; // enabled
+ buff[offset + 5] = channels;
+
+ buff[offset + 6] = codec >> 8;
+ buff[offset + 7] = codec;
+
+ buff[offset + 8] = kbps >> 8;
+ buff[offset + 9] = kbps;
+
+ sock._sQlen += 10;
+ sock.flush();
+ },
+
+ ReplitAudioRequestFrame(sock, channels, codec, kbps) {
+ const buff = sock._sQ;
+ const offset = sock._sQlen;
+
+ buff[offset] = 245; // msg-type
+ buff[offset + 1] = 1; // sub msg-type
+ buff[offset + 2] = 0;
+ buff[offset + 3] = 0; // length
+
+ sock._sQlen += 4;
+ sock.flush();
+ },
+
+ ReplitAudioEnableContinuousUpdate(sock) {
+ const buff = sock._sQ;
+ const offset = sock._sQlen;
+
+ buff[offset] = 245; // msg-type
+ buff[offset + 1] = 2; // sub msg-type
+ buff[offset + 2] = 0;
+ buff[offset + 3] = 0; // length
+
+ sock._sQlen += 4;
+ sock.flush();
+ },
+
QEMUExtendedKeyEvent(sock, keysym, down, keycode) {
function getRFBkeycode(xtScanCode) {
const upperByte = (keycode >> 8);
diff --git a/core/util/audio.js b/core/util/audio.js
new file mode 100644
index 00000000..dbfc4003
--- /dev/null
+++ b/core/util/audio.js
@@ -0,0 +1,201 @@
+/*
+ * noVNC: HTML5 VNC client
+ * Copyright (C) 2021 The noVNC Authors
+ * Licensed under MPL 2.0 (see LICENSE.txt)
+ *
+ * See README.md for usage and integration instructions.
+ */
+
+// The maximum allowable de-sync, in seconds. If the time between the last
+// received timestamp and the current audio playback timestamp exceeds this
+// value, the audio stream will be seeked to the most current timestamp
+// possible.
+const MAX_ALLOWABLE_DESYNC = 0.5;
+
+// The amount of time, in seconds, to keep in the audio buffer while seeking.
+// Whenever a de-sync event happens and we need to seek to a future
+// timestamp, we skip to the last buffered time minus this amount, so that the
+// browser has this amount of time worth of buffered audio data. This is done
+// to avoid having the browser enter a buffering state just after seeking.
+const SEEK_BUFFER_LENGTH = 0.2;
+
+// An audio stream built upon Media Stream Extensions.
+export default class AudioStream {
+ constructor(codec) {
+ this._codec = codec;
+ this._reset();
+ }
+
+ _reset() {
+ // Instantiate a media source and audio buffer/queue.
+ this._mediaSource = new MediaSource();
+ this._audioBuffer = null;
+ this._audioQ = [];
+
+ // Create a hidden audio element.
+ this._audio = document.createElement("audio");
+ this._audio.src = window.URL.createObjectURL(this._mediaSource);
+
+ // When data is queued, start playing.
+ this._audio.autoplay = true;
+ this._mediaSource.addEventListener(
+ "sourceopen",
+ this._onSourceOpen.bind(this),
+ false
+ );
+ this._audio.addEventListener(
+ "error",
+ (ev) => {
+ console.error("Audio element error", ev);
+ },
+ false
+ );
+ this._audio.addEventListener("canplay", () => {
+ try {
+ this._audio.play();
+ } catch (e) {
+ // Firefox and Chrome are totally cool with playing this
+ // the moment we can do it, but Safari throws an exception
+ // since play() is not called in a stack that ran a user
+ // event handler.
+ }
+ });
+ }
+
+ _onSourceOpen(e) {
+ if (this._audioBuffer) {
+ return;
+ }
+ this._audioBuffer = this._mediaSource.addSourceBuffer(this._codec);
+ this._audioBuffer.mode = "segments";
+ this._audioBuffer.addEventListener(
+ "updateend",
+ this._onUpdateBuffer.bind(this)
+ );
+ this._audioBuffer.addEventListener("error", (ev) => {
+ console.error("AudioBuffer error", ev);
+ });
+ }
+
+ _onUpdateBuffer() {
+ if (
+ !this._audioBuffer ||
+ this._audioBuffer.updating ||
+ this._audio.error
+ ) {
+ // The audio buffer is not yet ready to accept any new data.
+ return;
+ }
+ if (!this._audioQ.length) {
+ // There's nothing to append.
+ return;
+ }
+
+ const timestamp = this._audioQ[0][0];
+ if (this._audioQ.length === 1) {
+ this._appendChunk(timestamp, this._audioQ.pop()[1]);
+ return;
+ }
+
+ // If there is more than one chunk in the queue, they are coalesced
+ // into a single buffer. This is because following appendBuffer(),
+ // the audio buffer changes to an "updating" state for a small amount
+ // of time and any new chunks won't be able to be appended immediately.
+ // Since the internal queue is used when the browser is trying to catch
+ // up with the server, we want to have the audio buffer unappendable
+ // for a smaller amount of time.
+ let chunkLength = 0;
+ for (let i = 0; i < this._audioQ.length; ++i) {
+ chunkLength += this._audioQ[i][1].byteLength;
+ }
+ const chunk = new Uint8Array(chunkLength);
+ let offset = 0;
+ for (let i = 0; i < this._audioQ.length; ++i) {
+ chunk.set(new Uint8Array(this._audioQ[i][1]), offset);
+ offset += this._audioQ[i][1].byteLength;
+ }
+ this._audioQ.splice(0, this._audioQ.length);
+ this._appendChunk(timestamp, chunk);
+ }
+
+ // Append a chunk into the AudioBuffer. The caller should ensure that
+ // the AudioBuffer is ready to receive the chunk. If the difference
+ // between the current playback position of the audio and the timestamp
+ // exceeds the maximum allowable desync threshold, the audio will be
+ // seeked to the latest possible position that doesn't trigger buffering
+ // to avoid an arbitrarily large desync between video and audio.
+ _appendChunk(timestamp, chunk) {
+ this._audioBuffer.appendBuffer(chunk);
+ if (
+ timestamp - this._audio.currentTime > MAX_ALLOWABLE_DESYNC &&
+ (this._audio.seekable.length || this._audio.buffered.length)
+ ) {
+ console.debug("maximum allowable desync reached", {
+ readyState: this._audio.readyState,
+ buffered: (
+ (this._audio.buffered &&
+ this._audio.buffered.length &&
+ this._audio.buffered.end(
+ this._audio.buffered.length - 1
+ )) ||
+ 0
+ ).toFixed(2),
+ seekable: (
+ (this._audio.seekable &&
+ this._audio.seekable.length &&
+ this._audio.seekable.end(
+ this._audio.seekable.length - 1
+ )) ||
+ 0
+ ).toFixed(2),
+ time: this._audio.currentTime.toFixed(2),
+ delta: (timestamp - this._audio.currentTime).toFixed(2)
+ });
+ if (this._audio.buffered && this._audio.buffered.length) {
+ this._audio.currentTime =
+ this._audio.buffered.end(this._audio.buffered.length - 1) -
+ SEEK_BUFFER_LENGTH;
+ } else {
+ this._audio.currentTime =
+ this._audio.seekable.end(this._audio.seekable.length - 1) -
+ SEEK_BUFFER_LENGTH;
+ }
+ }
+ }
+
+ // Queues an audio chunk at a particular timestamp.
+ queueAudioFrame(timestamp, keyframe, chunk) {
+ // If the MSE audio buffer is not ready to receive the chunk or
+ // there are some other chunks waiting to be appended, we save
+ // a copy of it into our own internal queue. Eventually,
+ // when it becomes ready, we append all pending chunks at once.
+ if (
+ this._audioBuffer === null ||
+ this._audioBuffer.updating ||
+ this._audio.error ||
+ this._audioQ.length
+ ) {
+ // We need to make a copy, since `chunk` is a view of the underlying
+ // buffer owned by Websock, and will be mutated once we return.
+ // TODO: `keyframe` can be used to decide when to drop a chunk if
+ // there's enough backpressure.
+ const copy = new ArrayBuffer(chunk.byteLength);
+ new Uint8Array(copy).set(new Uint8Array(chunk));
+ this._audioQ.push([timestamp, copy]);
+ this._onUpdateBuffer();
+ return;
+ }
+
+ this._appendChunk(timestamp, chunk);
+ }
+
+ close() {
+ if (this._audio) {
+ this._audio.pause();
+ }
+ this._mediaSource = null;
+ this._audioBuffer = null;
+ this._audioQ = [];
+ this._audio = null;
+ }
+}
diff --git a/vnc.html b/vnc.html
index 24a118db..a7b79d1a 100644
--- a/vnc.html
+++ b/vnc.html
@@ -108,6 +108,11 @@
+
+
+