Re-apply https://github.com/replit/noVNC/pull/3 on top of v1.5.0

app/images/audio.svg

@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="25"
+   height="25"
+   viewBox="0 0 25 25"
+   id="svg2"
+   version="1.1"
+   inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)"
+   sodipodi:docname="audio.svg"
+   inkscape:export-filename="/home/ossman/devel/noVNC/images/keyboard.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90">
+  <defs
+     id="defs4" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#717171"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="22.627417"
+     inkscape:cx="-14.015068"
+     inkscape:cy="2.7882772"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     units="px"
+     inkscape:snap-bbox="true"
+     inkscape:bbox-paths="true"
+     inkscape:bbox-nodes="true"
+     inkscape:snap-bbox-edge-midpoints="true"
+     inkscape:snap-bbox-midpoints="false"
+     inkscape:window-width="2488"
+     inkscape:window-height="1376"
+     inkscape:window-x="72"
+     inkscape:window-y="27"
+     inkscape:window-maximized="1"
+     inkscape:object-paths="true"
+     inkscape:snap-intersection-paths="true"
+     inkscape:object-nodes="true"
+     inkscape:snap-midpoints="true"
+     inkscape:snap-smooth-nodes="true"
+     inkscape:document-rotation="0"
+     showguides="true"
+     inkscape:guide-bbox="true">
+    <inkscape:grid
+       type="xygrid"
+       id="grid4136" />
+  </sodipodi:namedview>
+  <metadata
+     id="metadata7">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-1027.3622)">
+    <path
+       style="color:#000000;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:medium;line-height:normal;font-family:sans-serif;font-variant-ligatures:normal;font-variant-position:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-alternates:normal;font-variant-east-asian:normal;font-feature-settings:normal;font-variation-settings:normal;text-indent:0;text-align:start;text-decoration:none;text-decoration-line:none;text-decoration-style:solid;text-decoration-color:#000000;letter-spacing:normal;word-spacing:normal;text-transform:none;writing-mode:lr-tb;direction:ltr;text-orientation:mixed;dominant-baseline:auto;baseline-shift:baseline;text-anchor:start;white-space:normal;shape-padding:0;shape-margin:0;inline-size:0;clip-rule:nonzero;display:inline;overflow:visible;visibility:visible;isolation:auto;mix-blend-mode:normal;color-interpolation:sRGB;color-interpolation-filters:linearRGB;solid-color:#000000;solid-opacity:1;vector-effect:none;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:1.58527;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;color-rendering:auto;image-rendering:auto;shape-rendering:auto;text-rendering:auto;enable-background:accumulate;stop-color:#000000;stop-opacity:1;opacity:1"
+       d="M 17.726562 1033.3848 A 0.79263502 0.79263502 0 0 0 17.142578 1033.6172 A 0.79263502 0.79263502 0 0 0 17.142578 1034.7383 C 18.501685 1036.0976 19.263676 1037.9389 19.263672 1039.8613 C 19.263677 1041.7837 18.501684 1043.627 17.142578 1044.9863 A 0.79263502 0.79263502 0 0 0 17.142578 1046.1074 A 0.79263502 0.79263502 0 0 0 18.263672 1046.1074 C 19.919622 1044.4512 20.849615 1042.2035 20.849609 1039.8613 C 20.849615 1037.5192 19.919622 1035.2734 18.263672 1033.6172 A 0.79263502 0.79263502 0 0 0 17.726562 1033.3848 z "
+       id="path1015" />
+    <path
+       id="rect854"
+       style="fill:#ffffff;stroke-width:0;stroke-dasharray:none"
+       d="m 11.211844,1034.336 a 0.79268394,0.79268394 0 0 0 -0.469698,0.1673 c -0.0022,0 -0.0059,9e-4 -0.0082,0 a 0.79268394,0.79268394 0 0 0 -0.0082,0.01 l -3.876241,3.2177 H 4.9546099 a 0.79268394,0.79268394 0 0 1 -0.0027,0 0.79268394,0.79268394 0 0 0 -0.0081,0 0.79268394,0.79268394 0 0 0 -0.7936268,0.7909 0.79268394,0.79268394 0 0 0 0,0.014 v 2.6455 c 0,0 0,0 0,0 a 0.79268394,0.79268394 0 0 0 0,0.01 0.79268394,0.79268394 0 0 0 0.7936268,0.7936 0.79268394,0.79268394 0 0 0 0.013503,0 h 1.8922867 l 3.7494814,3.1124 0.0054,0.01 a 0.79268394,0.79268394 0 0 0 0.610068,0.2862 0.79268394,0.79268394 0 0 0 0.790926,-0.7586 c 0.0026,-0.015 0.01355,-0.03 0.01355,-0.046 v -9.4561 a 0.79268394,0.79268394 0 0 0 -0.793627,-0.7936 0.79268394,0.79268394 0 0 0 -0.01355,0 z" />
+    <path
+       style="color:#000000;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:medium;line-height:normal;font-family:sans-serif;font-variant-ligatures:normal;font-variant-position:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-alternates:normal;font-variant-east-asian:normal;font-feature-settings:normal;font-variation-settings:normal;text-indent:0;text-align:start;text-decoration:none;text-decoration-line:none;text-decoration-style:solid;text-decoration-color:#000000;letter-spacing:normal;word-spacing:normal;text-transform:none;writing-mode:lr-tb;direction:ltr;text-orientation:mixed;dominant-baseline:auto;baseline-shift:baseline;text-anchor:start;white-space:normal;shape-padding:0;shape-margin:0;inline-size:0;clip-rule:nonzero;display:inline;overflow:visible;visibility:visible;isolation:auto;mix-blend-mode:normal;color-interpolation:sRGB;color-interpolation-filters:linearRGB;solid-color:#000000;solid-opacity:1;vector-effect:none;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:1.58527;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;color-rendering:auto;image-rendering:auto;shape-rendering:auto;text-rendering:auto;enable-background:accumulate;stop-color:#000000;stop-opacity:1;opacity:1"
+       d="M 15.794922 1035.3164 A 0.79263502 0.79263502 0 0 0 15.210938 1035.5488 A 0.79263502 0.79263502 0 0 0 15.210938 1036.6699 C 16.057983 1037.517 16.533213 1038.6634 16.533203 1039.8613 C 16.53319 1041.0592 16.057962 1042.2077 15.210938 1043.0547 A 0.79263502 0.79263502 0 0 0 15.210938 1044.1758 A 0.79263502 0.79263502 0 0 0 16.332031 1044.1758 C 17.475916 1043.0319 18.119123 1041.479 18.119141 1039.8613 C 18.119154 1038.2436 17.475944 1036.6928 16.332031 1035.5488 A 0.79263502 0.79263502 0 0 0 15.794922 1035.3164 z "
+       id="path1019" />
+    <path
+       style="color:#000000;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:medium;line-height:normal;font-family:sans-serif;font-variant-ligatures:normal;font-variant-position:normal;font-variant-caps:normal;font-variant-numeric:normal;font-variant-alternates:normal;font-variant-east-asian:normal;font-feature-settings:normal;font-variation-settings:normal;text-indent:0;text-align:start;text-decoration:none;text-decoration-line:none;text-decoration-style:solid;text-decoration-color:#000000;letter-spacing:normal;word-spacing:normal;text-transform:none;writing-mode:lr-tb;direction:ltr;text-orientation:mixed;dominant-baseline:auto;baseline-shift:baseline;text-anchor:start;white-space:normal;shape-padding:0;shape-margin:0;inline-size:0;clip-rule:nonzero;display:inline;overflow:visible;visibility:visible;isolation:auto;mix-blend-mode:normal;color-interpolation:sRGB;color-interpolation-filters:linearRGB;solid-color:#000000;solid-opacity:1;vector-effect:none;fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:1.58527;stroke-linecap:round;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;color-rendering:auto;image-rendering:auto;shape-rendering:auto;text-rendering:auto;enable-background:accumulate;stop-color:#000000;stop-opacity:1;opacity:1"
+       d="M 13.808594 1037.3027 A 0.79263502 0.79263502 0 0 0 13.224609 1037.5352 A 0.79263502 0.79263502 0 0 0 13.224609 1038.6562 C 13.544806 1038.9763 13.724616 1039.4089 13.724609 1039.8613 C 13.724598 1040.3137 13.544789 1040.7483 13.224609 1041.0684 A 0.79263502 0.79263502 0 0 0 13.224609 1042.1895 A 0.79263502 0.79263502 0 0 0 14.345703 1042.1895 C 14.962816 1041.5727 15.310525 1040.7338 15.310547 1039.8613 C 15.31056 1038.9888 14.962848 1038.152 14.345703 1037.5352 A 0.79263502 0.79263502 0 0 0 13.808594 1037.3027 z "
+       id="path1023" />
+  </g>
+</svg>

app/ui.js

@@ -232,6 +232,9 @@ const UI = {
         document.getElementById("noVNC_view_drag_button")
             .addEventListener('click', UI.toggleViewDrag);
 
+        document.getElementById("noVNC_audio_button")
+            .addEventListener('click', UI.toggleEnableAudio);
+
         document.getElementById("noVNC_control_bar_handle")
             .addEventListener('mousedown', UI.controlbarHandleMouseDown);
         document.getElementById("noVNC_control_bar_handle")
@@ -448,7 +451,7 @@ const UI = {
             UI.enableSetting('port');
             UI.enableSetting('path');
             UI.enableSetting('repeaterID');
-            UI.updatePowerButton();
+            UI.updateCapabilities();
             UI.keepControlbar();
         }
 
@@ -891,6 +894,24 @@ const UI = {
         }
     },
 
+    updateCapabilities() {
+        UI.updatePowerButton();
+        UI.updateAudioButton();
+    },
+
+    updateAudioButton() {
+        if (UI.connected &&
+            UI.rfb.capabilities.audio) {
+            document.getElementById('noVNC_audio_button')
+                .classList.remove("noVNC_hidden");
+            document.getElementById('noVNC_audio_button')
+                .classList.remove("noVNC_selected");
+        } else {
+            document.getElementById('noVNC_audio_button')
+                .classList.add("noVNC_hidden");
+        }
+    },
+
 /* ------^-------
  *   /SETTINGS
  * ==============
@@ -1059,7 +1080,7 @@ const UI = {
         UI.rfb.addEventListener("credentialsrequired", UI.credentials);
         UI.rfb.addEventListener("securityfailure", UI.securityFailed);
         UI.rfb.addEventListener("clippingviewport", UI.updateViewDrag);
-        UI.rfb.addEventListener("capabilities", UI.updatePowerButton);
+        UI.rfb.addEventListener("capabilities", UI.updateCapabilities);
         UI.rfb.addEventListener("clipboard", UI.clipboardReceive);
         UI.rfb.addEventListener("bell", UI.bell);
         UI.rfb.addEventListener("desktopname", UI.updateDesktopName);
@@ -1722,6 +1743,27 @@ const UI = {
         }
     },
 
+    toggleEnableAudio() {
+        if (!UI.rfb) return;
+
+        if (!document.getElementById('noVNC_audio_button')
+            .classList.contains("noVNC_selected")) {
+            UI.rfb.enableAudio(
+                2,
+                MediaSource.isTypeSupported('audio/webm;codecs=opus') ?
+                    RFB.audioCodecs.OpusWebM :
+                    RFB.audioCodecs.MP3,
+                32 * 1024  // 32kbps
+            );
+            document.getElementById('noVNC_audio_button')
+                .classList.add("noVNC_selected");
+        } else {
+            UI.rfb.disableAudio();
+            document.getElementById('noVNC_audio_button')
+                .classList.remove("noVNC_selected");
+        }
+    },
+
     updateShowDotCursor() {
         if (!UI.rfb) return;
         UI.rfb.showDotCursor = UI.getSetting('show_dot');

core/encodings.js

@@ -30,6 +30,7 @@ export const encodings = {
     pseudoEncodingContinuousUpdates: -313,
     pseudoEncodingCompressLevel9: -247,
     pseudoEncodingCompressLevel0: -256,
+    pseudoEncodingReplitAudio: 0x52706c41,
     pseudoEncodingVMwareCursor: 0x574d5664,
     pseudoEncodingExtendedClipboard: 0xc0a1e5ce
 };

core/rfb.js

@@ -13,6 +13,7 @@ import { encodeUTF8, decodeUTF8 } from './util/strings.js';
 import { dragThreshold } from './util/browser.js';
 import { clientToElement } from './util/element.js';
 import { setCapture } from './util/events.js';
+import AudioStream from './util/audio.js';
 import EventTargetMixin from './util/eventtarget.js';
 import Display from "./display.js";
 import Inflator from "./inflator.js";
@@ -137,7 +138,7 @@ export default class RFB extends EventTargetMixin {
 
         this._fbName = "";
 
-        this._capabilities = { power: false };
+        this._capabilities = { power: false, audio: false };
 
         this._supportsFence = false;
 
@@ -149,6 +150,8 @@ export default class RFB extends EventTargetMixin {
         this._screenFlags = 0;
 
         this._qemuExtKeyEventSupported = false;
+        this._replitAudioSupported = false;
+        this._replitAudioServerVersion = -1;
 
         this._clipboardText = null;
         this._clipboardServerCapabilitiesActions = {};
@@ -195,6 +198,11 @@ export default class RFB extends EventTargetMixin {
         this._gestureLastMagnitudeX = 0;
         this._gestureLastMagnitudeY = 0;
 
+        // Audio state
+        this._audioEnabled = false;
+        this._audioMimeType = null;
+        this._audioStream = null;
+
         // Bound event handlers
         this._eventHandlers = {
             focusCanvas: this._focusCanvas.bind(this),
@@ -541,6 +549,25 @@ export default class RFB extends EventTargetMixin {
         return this._display.toBlob(callback, type, quality);
     }
 
+    enableAudio(channels, codec, kbps) {
+        if (this._audioEnabled) { return; }
+
+        this._audioEnabled = true;
+        if (codec == RFB.audioCodecs.OpusWebM) {
+            this._audioMimeType = 'audio/webm;codecs=opus';
+        } else if (codec == RFB.audioCodecs.MP3) {
+            this._audioMimeType = 'audio/mpeg';
+        }
+        RFB.messages.ReplitAudioStartEncoder(this._sock, true, channels, codec, kbps);
+    }
+
+    disableAudio() {
+        if (!this._audioEnabled) { return; }
+
+        this._audioEnabled = false;
+        RFB.messages.ReplitAudioStartEncoder(this._sock, false, 0, 0, 0);
+    }
+
     // ===== PRIVATE METHODS =====
 
     _connect() {
@@ -2132,6 +2159,7 @@ export default class RFB extends EventTargetMixin {
         encs.push(encodings.pseudoEncodingLastRect);
         encs.push(encodings.pseudoEncodingQEMUExtendedKeyEvent);
         encs.push(encodings.pseudoEncodingQEMULedEvent);
+        encs.push(encodings.pseudoEncodingReplitAudio);
         encs.push(encodings.pseudoEncodingExtendedDesktopSize);
         encs.push(encodings.pseudoEncodingXvp);
         encs.push(encodings.pseudoEncodingFence);
@@ -2411,6 +2439,54 @@ export default class RFB extends EventTargetMixin {
         return true;
     }
 
+    _handleReplitAudioPseudoEncodingMsg() {
+        if (this._sock.rQwait("Repl.it audio message", 3, 1)) { return false; }
+        const submessage = this._sock.rQshift8();
+        const length = this._sock.rQshift16();
+        if (this._sock.rQwait("Repl.it audio message", length, 4)) { return false; }
+
+        switch (submessage) {
+            case 0: { // StartCapture response.
+                const enabled = this._sock.rQshift8() == 1;
+
+                if (enabled) {
+                    this._audioStream = new AudioStream(this._audioMimeType);
+                    RFB.messages.ReplitAudioEnableContinuousUpdate(this._sock);
+                } else if (this._audioStream != null) {
+                    this._audioStream.close();
+                    this._audioStream = null;
+                }
+                break;
+            }
+
+            case 1: { // AudioFrame response.
+                const keyframeAndTimestamp = this._sock.rQshift32();
+                const keyframe = (keyframeAndTimestamp & 0x80000000) != 0;
+                const timestamp = keyframeAndTimestamp & 0x7fffffff;
+                const data = this._sock.rQshiftBytes(length - 4);
+                if (this._audioStream != null) {
+                    this._audioStream.queueAudioFrame(timestamp / 1000, keyframe, data);
+                }
+                break;
+            }
+
+            case 2: { // StartContinuousUpdates response.
+                const enabled = this._sock.rQshift8() == 1;
+                if (!enabled && this._audioStream != null) {
+                    this._audioStream.close();
+                    this._audioStream = null;
+                }
+                break;
+            }
+
+            default:
+                this._fail("Illegal server Repl.it audio message (msg: " + submessage + ")");
+                break;
+        }
+
+        return true;
+    }
+
     _handleXvpMsg() {
         if (this._sock.rQwait("XVP version and message", 3, 1)) { return false; }
         this._sock.rQskipBytes(1);  // Padding
@@ -2479,6 +2555,9 @@ export default class RFB extends EventTargetMixin {
                 }
                 return true;
 
+            case 245: // Repl.it audio message
+                return this._handleReplitAudioPseudoEncodingMsg();
+
             case 248: // ServerFence
                 return this._handleServerFenceMsg();
 
@@ -2557,6 +2636,9 @@ export default class RFB extends EventTargetMixin {
                 this._qemuExtKeyEventSupported = true;
                 return true;
 
+            case encodings.pseudoEncodingReplitAudio:
+                return this._handleReplitAudioPseudoEncoding();
+
             case encodings.pseudoEncodingDesktopName:
                 return this._handleDesktopName();
 
@@ -2728,6 +2810,25 @@ export default class RFB extends EventTargetMixin {
         return true;
     }
 
+    _handleReplitAudioPseudoEncoding() {
+        if (this._sock.rQwait("Repl.it audio", 4)) {
+            return false;
+        }
+
+        const version = this._sock.rQshift16();
+        const codecs = this._sock.rQshift16();
+
+        if (this._sock.rQwait("Repl.it audio", 2 * codecs, 4)) {
+            return false;
+        }
+        this._sock.rQshiftStr(2 * codecs);
+
+        this._replitAudioSupported = true;
+        this._replitAudioServerVersion = version;
+        this._setCapability("audio", true);
+        return true;
+    }
+
     _handleDesktopName() {
         if (this._sock.rQwait("DesktopName", 4)) {
             return false;
@@ -2935,6 +3036,12 @@ export default class RFB extends EventTargetMixin {
     }
 }
 
+// Audio codecs
+RFB.audioCodecs = {
+    OpusWebM: 0,
+    MP3: 1,
+};
+
 // Class Methods
 RFB.messages = {
     keyEvent(sock, keysym, down) {
@@ -2948,6 +3055,54 @@ RFB.messages = {
         sock.flush();
     },
 
+    ReplitAudioStartEncoder(sock, enabled, channels, codec, kbps) {
+        const buff = sock._sQ;
+        const offset = sock._sQlen;
+
+        buff[offset] = 245;   // msg-type
+        buff[offset + 1] = 0; // sub msg-type
+        buff[offset + 2] = 0;
+        buff[offset + 3] = 6; // length
+
+        buff[offset + 4] = enabled ? 1 : 0; // enabled
+        buff[offset + 5] = channels;
+
+        buff[offset + 6] = codec >> 8;
+        buff[offset + 7] = codec;
+
+        buff[offset + 8] = kbps >> 8;
+        buff[offset + 9] = kbps;
+
+        sock._sQlen += 10;
+        sock.flush();
+    },
+
+    ReplitAudioRequestFrame(sock, channels, codec, kbps) {
+        const buff = sock._sQ;
+        const offset = sock._sQlen;
+
+        buff[offset] = 245;   // msg-type
+        buff[offset + 1] = 1; // sub msg-type
+        buff[offset + 2] = 0;
+        buff[offset + 3] = 0; // length
+
+        sock._sQlen += 4;
+        sock.flush();
+    },
+
+    ReplitAudioEnableContinuousUpdate(sock) {
+        const buff = sock._sQ;
+        const offset = sock._sQlen;
+
+        buff[offset] = 245;        // msg-type
+        buff[offset + 1] = 2;      // sub msg-type
+        buff[offset + 2] = 0;
+        buff[offset + 3] = 0; // length
+
+        sock._sQlen += 4;
+        sock.flush();
+    },
+
     QEMUExtendedKeyEvent(sock, keysym, down, keycode) {
         function getRFBkeycode(xtScanCode) {
             const upperByte = (keycode >> 8);

core/util/audio.js

@@ -0,0 +1,201 @@
+/*
+ * noVNC: HTML5 VNC client
+ * Copyright (C) 2021 The noVNC Authors
+ * Licensed under MPL 2.0 (see LICENSE.txt)
+ *
+ * See README.md for usage and integration instructions.
+ */
+
+// The maximum allowable de-sync, in seconds. If the time between the last
+// received timestamp and the current audio playback timestamp exceeds this
+// value, the audio stream will be seeked to the most current timestamp
+// possible.
+const MAX_ALLOWABLE_DESYNC = 0.5;
+
+// The amount of time, in seconds, to keep in the audio buffer while seeking.
+// Whenever a de-sync event happens and we need to seek to a future
+// timestamp, we skip to the last buffered time minus this amount, so that the
+// browser has this amount of time worth of buffered audio data. This is done
+// to avoid having the browser enter a buffering state just after seeking.
+const SEEK_BUFFER_LENGTH = 0.2;
+
+// An audio stream built upon Media Stream Extensions.
+export default class AudioStream {
+    constructor(codec) {
+        this._codec = codec;
+        this._reset();
+    }
+
+    _reset() {
+        // Instantiate a media source and audio buffer/queue.
+        this._mediaSource = new MediaSource();
+        this._audioBuffer = null;
+        this._audioQ = [];
+
+        // Create a hidden audio element.
+        this._audio = document.createElement("audio");
+        this._audio.src = window.URL.createObjectURL(this._mediaSource);
+
+        // When data is queued, start playing.
+        this._audio.autoplay = true;
+        this._mediaSource.addEventListener(
+            "sourceopen",
+            this._onSourceOpen.bind(this),
+            false
+        );
+        this._audio.addEventListener(
+            "error",
+            (ev) => {
+                console.error("Audio element error", ev);
+            },
+            false
+        );
+        this._audio.addEventListener("canplay", () => {
+            try {
+                this._audio.play();
+            } catch (e) {
+                // Firefox and Chrome are totally cool with playing this
+                // the moment we can do it, but Safari throws an exception
+                // since play() is not called in a stack that ran a user
+                // event handler.
+            }
+        });
+    }
+
+    _onSourceOpen(e) {
+        if (this._audioBuffer) {
+            return;
+        }
+        this._audioBuffer = this._mediaSource.addSourceBuffer(this._codec);
+        this._audioBuffer.mode = "segments";
+        this._audioBuffer.addEventListener(
+            "updateend",
+            this._onUpdateBuffer.bind(this)
+        );
+        this._audioBuffer.addEventListener("error", (ev) => {
+            console.error("AudioBuffer error", ev);
+        });
+    }
+
+    _onUpdateBuffer() {
+        if (
+            !this._audioBuffer ||
+            this._audioBuffer.updating ||
+            this._audio.error
+        ) {
+            // The audio buffer is not yet ready to accept any new data.
+            return;
+        }
+        if (!this._audioQ.length) {
+            // There's nothing to append.
+            return;
+        }
+
+        const timestamp = this._audioQ[0][0];
+        if (this._audioQ.length === 1) {
+            this._appendChunk(timestamp, this._audioQ.pop()[1]);
+            return;
+        }
+
+        // If there is more than one chunk in the queue, they are coalesced
+        // into a single buffer. This is because following appendBuffer(),
+        // the audio buffer changes to an "updating" state for a small amount
+        // of time and any new chunks won't be able to be appended immediately.
+        // Since the internal queue is used when the browser is trying to catch
+        // up with the server, we want to have the audio buffer unappendable
+        // for a smaller amount of time.
+        let chunkLength = 0;
+        for (let i = 0; i < this._audioQ.length; ++i) {
+            chunkLength += this._audioQ[i][1].byteLength;
+        }
+        const chunk = new Uint8Array(chunkLength);
+        let offset = 0;
+        for (let i = 0; i < this._audioQ.length; ++i) {
+            chunk.set(new Uint8Array(this._audioQ[i][1]), offset);
+            offset += this._audioQ[i][1].byteLength;
+        }
+        this._audioQ.splice(0, this._audioQ.length);
+        this._appendChunk(timestamp, chunk);
+    }
+
+    // Append a chunk into the AudioBuffer. The caller should ensure that
+    // the AudioBuffer is ready to receive the chunk. If the difference
+    // between the current playback position of the audio and the timestamp
+    // exceeds the maximum allowable desync threshold, the audio will be
+    // seeked to the latest possible position that doesn't trigger buffering
+    // to avoid an arbitrarily large desync between video and audio.
+    _appendChunk(timestamp, chunk) {
+        this._audioBuffer.appendBuffer(chunk);
+        if (
+            timestamp - this._audio.currentTime > MAX_ALLOWABLE_DESYNC &&
+            (this._audio.seekable.length || this._audio.buffered.length)
+        ) {
+            console.debug("maximum allowable desync reached", {
+                readyState: this._audio.readyState,
+                buffered: (
+                    (this._audio.buffered &&
+                        this._audio.buffered.length &&
+                        this._audio.buffered.end(
+                            this._audio.buffered.length - 1
+                        )) ||
+                    0
+                ).toFixed(2),
+                seekable: (
+                    (this._audio.seekable &&
+                        this._audio.seekable.length &&
+                        this._audio.seekable.end(
+                            this._audio.seekable.length - 1
+                        )) ||
+                    0
+                ).toFixed(2),
+                time: this._audio.currentTime.toFixed(2),
+                delta: (timestamp - this._audio.currentTime).toFixed(2)
+            });
+            if (this._audio.buffered && this._audio.buffered.length) {
+                this._audio.currentTime =
+                    this._audio.buffered.end(this._audio.buffered.length - 1) -
+                    SEEK_BUFFER_LENGTH;
+            } else {
+                this._audio.currentTime =
+                    this._audio.seekable.end(this._audio.seekable.length - 1) -
+                    SEEK_BUFFER_LENGTH;
+            }
+        }
+    }
+
+    // Queues an audio chunk at a particular timestamp.
+    queueAudioFrame(timestamp, keyframe, chunk) {
+        // If the MSE audio buffer is not ready to receive the chunk or
+        // there are some other chunks waiting to be appended, we save
+        // a copy of it into our own internal queue. Eventually,
+        // when it becomes ready, we append all pending chunks at once.
+        if (
+            this._audioBuffer === null ||
+            this._audioBuffer.updating ||
+            this._audio.error ||
+            this._audioQ.length
+        ) {
+            // We need to make a copy, since `chunk` is a view of the underlying
+            // buffer owned by Websock, and will be mutated once we return.
+            // TODO: `keyframe` can be used to decide when to drop a chunk if
+            // there's enough backpressure.
+            const copy = new ArrayBuffer(chunk.byteLength);
+            new Uint8Array(copy).set(new Uint8Array(chunk));
+            this._audioQ.push([timestamp, copy]);
+            this._onUpdateBuffer();
+            return;
+        }
+
+        this._appendChunk(timestamp, chunk);
+    }
+
+    close() {
+        if (this._audio) {
+            this._audio.pause();
+        }
+        this._mediaSource = null;
+        this._audioBuffer = null;
+        this._audioQ = [];
+        this._audio = null;
+    }
+}

vnc.html

@@ -108,6 +108,11 @@
             </div>
             </div>
 
+            <!-- Audio -->
+            <input type="image" alt="Toggle Audio" src="app/images/audio.svg"
+                id="noVNC_audio_button" class="noVNC_button noVNC_hidden"
+                title="Toggle Audio">
+
             <!-- Shutdown/Reboot -->
             <input type="image" alt="Shutdown/Reboot" src="app/images/power.svg"
                 id="noVNC_power_button" class="noVNC_button"