diff --git a/app/ui.js b/app/ui.js index 529fdbea..6b5e6b76 100644 --- a/app/ui.js +++ b/app/ui.js @@ -41,6 +41,7 @@ import Keyboard from "../core/input/keyboard.js"; import RFB from "../core/rfb.js"; import { MouseButtonMapper, XVNC_BUTTONS } from "../core/mousebuttonmapper.js"; import * as WebUtil from "./webutil.js"; +import { uuidv4 } from '../core/util/strings.js'; const PAGE_TITLE = "KasmVNC"; @@ -70,7 +71,8 @@ const UI = { selectedMonitor: null, refreshRotation: 0, currentDisplay: null, - displayWindows: ['primary'], + displayWindows: new Map([['primary', 'primary']]), + registeredWindows: new Map([['primary', 'primary']]), supportsBroadcastChannel: (typeof BroadcastChannel !== "undefined"), @@ -177,11 +179,11 @@ const UI = { UI.hideKeyboardControls(); } }); - - window.addEventListener("unload", (e) => { - if (UI.rfb) { - UI.disconnect(); - } + + window.addEventListener("unload", (e) => { + if (UI.rfb) { + UI.disconnect(); + } }); return Promise.resolve(UI.rfb); @@ -378,7 +380,7 @@ const UI = { .pointerEvents({ holdDuration: 350 }) .on("hold", (e) => { const buttonsEl = document.querySelector(".keyboard-controls"); - + const isOpen = buttonsEl.classList.contains("is-open"); buttonsEl.classList.toggle("was-open", isOpen); buttonsEl.classList.toggle("is-open", !isOpen); @@ -630,7 +632,7 @@ const UI = { UI.addClickHandle('noVNC_identify_monitors_button', UI._identify); UI.addClickHandle('noVNC_addMonitor', UI.addSecondaryMonitor); UI.addClickHandle('noVNC_refreshMonitors', UI.displaysRefresh); - + } }, @@ -657,7 +659,7 @@ const UI = { isControlPanelItemClick(e) { if (!(e && e.target && e.target.classList && e.target.parentNode && ( - e.target.classList.contains('noVNC_button') && e.target.parentNode.id !== 'noVNC_modifiers' || + e.target.classList.contains('noVNC_button') && e.target.parentNode.id !== 'noVNC_modifiers' || e.target.classList.contains('noVNC_button_div') || e.target.classList.contains('noVNC_heading') ) @@ -677,7 +679,7 @@ const UI = { document.documentElement.classList.remove("noVNC_disconnected"); const transitionElem = document.getElementById("noVNC_transition_text"); - if (WebUtil.isInsideKasmVDI()) + if (WebUtil.isInsideKasmVDI()) { parent.postMessage({ action: 'connection_state', value: state}, '*' ); } @@ -753,7 +755,7 @@ const UI = { document.getElementById("noVNC_connection_stats").style.visibility = "hidden"; UI.statsInterval = null; } - + }, threading() { @@ -1459,10 +1461,10 @@ const UI = { UI.rfb = new RFB(document.getElementById('noVNC_container'), document.getElementById('noVNC_keyboardinput'), url, - { + { shared: UI.getSetting('shared'), repeaterID: UI.getSetting('repeaterID'), - credentials: { password: password } + credentials: { password: password } }, true ); UI.rfb.addEventListener("connect", UI.connectFinished); @@ -1521,9 +1523,9 @@ const UI = { .catch(() => {}); } // KASM-960 workaround, disable seamless on Safari - if (/^((?!chrome|android).)*safari/i.test(navigator.userAgent)) - { - UI.rfb.clipboardSeamless = false; + if (/^((?!chrome|android).)*safari/i.test(navigator.userAgent)) + { + UI.rfb.clipboardSeamless = false; } UI.rfb.preferLocalCursor = UI.getSetting('prefer_local_cursor'); UI.rfb.enableWebP = UI.getSetting('enable_webp'); @@ -1540,7 +1542,7 @@ const UI = { window.attachEvent('onload', WindowLoad); window.attachEvent('message', UI.receiveMessage); } - if (UI.rfb.clipboardDown){ + if (UI.rfb.clipboardDown){ UI.rfb.addEventListener("clipboard", UI.clipboardRx); } UI.rfb.addEventListener("disconnect", UI.disconnectedRx); @@ -1553,7 +1555,7 @@ const UI = { UI._sessionTimeoutInterval = setInterval(function() { if (UI.rfb) { const timeSinceLastActivityInS = (Date.now() - UI.rfb.lastActiveAt) / 1000; - let idleDisconnectInS = 1200; //20 minute default + let idleDisconnectInS = 1200; //20 minute default if (Number.isFinite(parseFloat(UI.rfb.idleDisconnect))) { idleDisconnectInS = parseFloat(UI.rfb.idleDisconnect) * 60; } @@ -2009,8 +2011,9 @@ const UI = { }, async addSecondaryMonitor() { - let new_display_path = window.location.pathname.replace(/[^/]*$/, '') - let new_display_url = `${window.location.protocol}//${window.location.host}${new_display_path}screen.html`; + let new_display_path = window.location.pathname.replace(/[^/]*$/, ''); + const windowId = uuidv4(); + let new_display_url = `${window.location.protocol}//${window.location.host}${new_display_path}screen.html?windowId=${windowId}`; const auto_placement = document.getElementById('noVNC_auto_placement').checked if (auto_placement && 'getScreenDetails' in window) { @@ -2020,11 +2023,11 @@ const UI = { permission = (state === 'granted' || state === 'prompt'); if (permission && window.screen.isExtended) { const details = await window.getScreenDetails() - const current = UI.increaseCurrentDisplay(details) + const current = UI.increaseCurrentDisplay(details) let screen = details.screens[current] const options = 'left='+screen.availLeft+',top='+screen.availTop+',width='+screen.availWidth+',height='+screen.availHeight+',fullscreen' let newdisplay = window.open(new_display_url, '_blank', options); - UI.displayWindows.push(newdisplay); + UI.displayWindows.set(windowId, newdisplay); return; } } catch (e) { @@ -2032,17 +2035,19 @@ const UI = { // Nothing. } } - + Log.Debug(`Opening a secondary display ${new_display_url}`) let newdisplay = window.open(new_display_url, '_blank', 'toolbar=0,location=0,menubar=0'); - UI.displayWindows.push(newdisplay); + if (newdisplay) { + UI.displayWindows.set(windowId, newdisplay); + } }, initMonitors(screenPlan) { const { scale } = UI.multiMonitorSettings() let monitors = [] let showNativeResolution = false - let num = 1 + let num = 1; screenPlan.screens.forEach(screen => { if (parseFloat(screen.pixelRatio) != 1) { showNativeResolution = true @@ -2078,7 +2083,7 @@ const UI = { }, updateMonitors(screenPlan) { - UI.initMonitors(screenPlan) + UI.initMonitors(screenPlan) UI.recenter() UI.draw() }, @@ -2132,7 +2137,7 @@ const UI = { prev = monitors[i] } } - }, + }, rect(ctx, x, y, w, h) { ctx.beginPath(); @@ -2223,7 +2228,9 @@ const UI = { serverWidth: Math.round(width * scale), screens } - UI.rfb.applyScreenPlan(screenPlan); + if (UI.rfb) { + UI.rfb.applyScreenPlan(screenPlan); + } }, @@ -2238,7 +2245,7 @@ const UI = { let dragok = false let startX; let startY; - + offsetX = bb.left offsetY = bb.top @@ -2306,7 +2313,7 @@ const UI = { var dx = mx - startX; var dy = my - startY; - // move each rect that isDragging + // move each rect that isDragging // by the distance the mouse has moved // since the last mousemove for (var i = 0; i < monitors.length; i++) { @@ -2910,7 +2917,7 @@ const UI = { UI.showControlInput("noVNC_keyboard_button"); UI.showControlInput("noVNC_toggle_extra_keys_button"); UI.showControlInput("noVNC_clipboard_button"); - UI.showControlInput("noVNC_game_mode_button"); + UI.showControlInput("noVNC_game_mode_button"); } }, @@ -2939,7 +2946,7 @@ const UI = { UI.closeControlbar(); UI.showStatus('Press Esc Key to Exit Pointer Lock Mode', 'warn', 5000, true); } else { - //If in game mode + //If in game mode if (UI.rfb.pointerRelative) { UI.showStatus('Game Mode paused, click on screen to resume Game Mode.', 'warn', 5000, true); } else { @@ -2982,7 +2989,7 @@ const UI = { screenRegistered(e) { console.log('screen registered') - + // Get the current screen plan // When a new display is added, it is defaulted to be placed to the far right relative to existing displays and to the top if (UI.rfb) { @@ -2999,7 +3006,7 @@ const UI = { UI.updateMonitors(screenPlan) UI._identify(UI.monitors) } - + }, //Helper to add options to dropdown. diff --git a/core/display.js b/core/display.js index 3e38f7cd..9be9aaa9 100644 --- a/core/display.js +++ b/core/display.js @@ -100,7 +100,7 @@ export default class Display { this._fps = 0; this._isPrimaryDisplay = isPrimaryDisplay; this._screenID = uuidv4(); - this._screens = [{ + this._screens = [{ screenID: this._screenID, screenIndex: 0, width: this._target.width, //client @@ -150,7 +150,7 @@ export default class Display { this._enableCanvasBuffer = value; - + if (value && this._target) { //copy current visible canvas to backbuffer @@ -175,9 +175,9 @@ export default class Display { if (!this._isPrimaryDisplay && this._screens[0].screenIndex == 0) { return -1; } - return this._screens[0].screenIndex; + return this._screens[0].screenIndex; } - + get antiAliasing() { return this._antiAliasing; } set antiAliasing(value) { this._antiAliasing = value; @@ -228,7 +228,7 @@ export default class Display { */ getClientRelativeCoordinates(x, y) { for (let i = 0; i < this._screens.length; i++) { - if ( + if ( (x >= this._screens[i].x && x <= this._screens[i].x + this._screens[i].serverWidth) && (y >= this._screens[i].y && y <= this._screens[i].y + this._screens[i].serverHeight) ) @@ -242,7 +242,7 @@ export default class Display { } } - /* + /* Returns coordinates that are server relative when multiple monitors are in use */ getServerRelativeCoordinates(screenIndex, x, y) { @@ -263,9 +263,9 @@ export default class Display { let i = 0; - + //getting parent node size with sub-pixel precision - let parentNodeSize = this._target.parentNode.getBoundingClientRect(); + let parentNodeSize = this._target.parentNode.getBoundingClientRect(); //recalculate primary display container size this._screens[i].containerHeight = Math.floor(parentNodeSize.height / 2) * 2; this._screens[i].containerWidth = Math.floor(parentNodeSize.width / 2) * 2; @@ -284,7 +284,7 @@ export default class Display { ( disableScaling || (this._screens[i].serverReportedWidth !== this._screens[i].serverWidth || this._screens[i].serverReportedHeight !== this._screens[i].serverHeight) - ) && + ) && (!max_width && !max_height) ) { height = this._screens[i].serverReportedHeight; @@ -307,7 +307,7 @@ export default class Display { } //physically small device with high DPI else if (this._antiAliasing === 0 && this._screens[i].pixelRatio > 1 && width < 1000 & width > 0) { - Log.Info('Device Pixel ratio: ' + this._screens[i].pixelRatio + ' Reported Resolution: ' + width + 'x' + height); + Log.Info('Device Pixel ratio: ' + this._screens[i].pixelRatio + ' Reported Resolution: ' + width + 'x' + height); let targetDevicePixelRatio = 1.5; if (this._screens[i].pixelRatio > 2) { targetDevicePixelRatio = 2; } let scaledWidth = (width * this._screens[i].pixelRatio) * (1 / targetDevicePixelRatio); @@ -317,10 +317,10 @@ export default class Display { scale = 1 / scaleRatio; Log.Info('Small device with hDPI screen detected, auto scaling at ' + scaleRatio + ' to ' + width + 'x' + height); } - + let clientServerRatioH = this._screens[i].containerHeight / height; let clientServerRatioW = this._screens[i].containerWidth / width; - + this._screens[i].height = Math.floor(height * clientServerRatioH); this._screens[i].width = Math.floor(width * clientServerRatioW); this._screens[i].serverWidth = width; @@ -376,12 +376,12 @@ export default class Display { return changes; } - addScreen(screenID, width, height, pixelRatio, containerHeight, containerWidth, scale, serverWidth, serverHeight, x, y) { + addScreen(screenID, width, height, pixelRatio, containerHeight, containerWidth, scale, serverWidth, serverHeight, x, y, windowId) { if (!this._isPrimaryDisplay) { throw new Error("Cannot add a screen to a secondary display."); } else if (containerHeight === 0 || containerWidth === 0 || pixelRatio === 0) { - Log.Warn("Invalid screen configuration."); + Log.Warn("Invalid screen configuration."); } let screenIdx = -1; @@ -395,8 +395,8 @@ export default class Display { if (screenIdx > 0) { //existing screen, update const existing_screen = this._screens[screenIdx]; - if (existing_screen.serverHeight !== serverHeight || existing_screen.serverWidth !== serverWidth || existing_screen.width !== width || existing_screen.height !== height - || existing_screen.containerHeight !== containerHeight || existing_screen.containerWidth !== containerWidth || existing_screen.scale !== scale || existing_screen.pixelRatio !== pixelRatio || + if (existing_screen.serverHeight !== serverHeight || existing_screen.serverWidth !== serverWidth || existing_screen.width !== width || existing_screen.height !== height + || existing_screen.containerHeight !== containerHeight || existing_screen.containerWidth !== containerWidth || existing_screen.scale !== scale || existing_screen.pixelRatio !== pixelRatio || existing_screen.x !== x || existing_screen.y !== y) { existing_screen.width = width; existing_screen.height = height; @@ -432,14 +432,18 @@ export default class Display { pixelRatio: pixelRatio, containerHeight: containerHeight, containerWidth: containerWidth, - channel: UI.displayWindows[this.screens.length], + channel: UI.displayWindows.get(windowId), scale: scale, x2: x + serverWidth, y2: serverHeight } this._screens.push(new_screen); - new_screen.channel.postMessage({ eventType: "registered", screenIndex: new_screen.screenIndex }); + if (new_screen.channel) { + UI.registeredWindows.set(screenID, windowId); + new_screen.channel.postMessage({eventType: "registered", screenIndex: new_screen.screenIndex}); + } else + Log.Debug(`Channel not found for screenId ${screenID}`); return new_screen.screenIndex; } @@ -454,7 +458,11 @@ export default class Display { if (this._screens[i].screenID == screenID) { //flush all rects on target screen this._flushRectsScreen(i); - UI.displayWindows.splice(i, 1); + const windowId = UI.registeredWindows.get(screenID); + if (windowId) { + UI.registeredWindows.delete(screenID); + UI.displayWindows.delete(windowId); + } this._screens.splice(i, 1); removed = true; break; @@ -577,7 +585,7 @@ export default class Display { let canvas = this._backbuffer; if (canvas == undefined) { return; } - + if (this._screens.length > 0) { width = this._screens[0].serverWidth; height = this._screens[0].serverHeight; @@ -603,7 +611,7 @@ export default class Display { } } - + // Readjust the viewport as it may be incorrectly sized // and positioned @@ -648,7 +656,7 @@ export default class Display { if (onflush_message) this.onflush(); } - + /* * Clears the buffer of anything that has not yet been displayed. * This must be called when switching between transit modes tcp/udp @@ -743,7 +751,7 @@ export default class Display { if ((typeof ImageDecoder !== 'undefined') && (this._threading)) { let imageDecoder = new ImageDecoder({ data: arr, type: mime }); let rect = { - 'type': 'vid', + 'type': 'vid', 'img': null, 'x': x, 'y': y, @@ -757,30 +765,22 @@ export default class Display { return; } - let rect = { - 'type': 'img', + const blob = new Blob([arr], { type: mime }); + const rect = { + 'type': 'bitmap', 'img': null, 'x': x, 'y': y, 'width': width, 'height': height, - 'frame_id': frame_id - } + 'frame_id': frame_id, + 'mime': mime + }; this._processRectScreens(rect); - - if (rect.inPrimary) { - const img = new Image(); - img.src = "data: " + mime + ";base64," + Base64.encode(arr); - rect.img = img; - } else { - rect.type = "_img"; - } - if (rect.inSecondary) { - rect.mime = mime; - rect.src = "data: " + mime + ";base64," + Base64.encode(arr); - } - - this._asyncRenderQPush(rect); + createImageBitmap(blob).then((bitmapImg) => { + rect.img = bitmapImg; + this._asyncRenderQPush(rect); + }); } transparentRect(x, y, width, height, img, frame_id, hashId) { @@ -832,8 +832,8 @@ export default class Display { if (!fromQueue) { var buf; if (!ArrayBuffer.isView(arr)) { - buf = arr; - } else { + buf = arr; + } else { buf = arr.buffer; } // NB(directxman12): it's technically more performant here to use preallocated arrays, @@ -869,9 +869,9 @@ export default class Display { this._drawCtx.putImageData(img, x, y); } else { this._targetCtx.putImageData(img, x, y); - + } - + } } @@ -1027,6 +1027,10 @@ export default class Display { this.drawImage(a.img, pos.x, pos.y, a.width, a.height); a.img.close(); break; + case 'bitmap': + this.drawImage(a.img, pos.x, pos.y, a.width, a.height); + a.img.close(); + break; default: this._syncFrameQueue.shift(); continue; @@ -1100,7 +1104,7 @@ export default class Display { this._asyncFrameQueue[frameIx][1] += rect.rect_cnt; if (rect.rect_cnt == 0) { Log.Warn("Invalid rect count"); - } + } } if (this._asyncFrameQueue[frameIx][1] > 0 && this._asyncFrameQueue[frameIx][2].length >= this._asyncFrameQueue[frameIx][1]) { @@ -1125,13 +1129,13 @@ export default class Display { this._asyncFrameQueue.shift(); this._droppedFrames += (rect.frame_id - newestFrameID); } - + let rect_cnt = ((rect.type == "flip") ? rect.rect_cnt : 0); this._asyncFrameQueue.push([ rect.frame_id, rect_cnt, [ rect ], (rect_cnt == 1), 0, 0 ]); - + } } - + } /* @@ -1166,7 +1170,7 @@ export default class Display { Log.Warn("Frame has more rects than the reported rect_cnt."); } } - while (currentFrameRectIx < this._asyncFrameQueue[frameIx][2].length) { + while (currentFrameRectIx < this._asyncFrameQueue[frameIx][2].length) { if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img') { if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) { this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type = 'skip'; @@ -1210,10 +1214,10 @@ export default class Display { let secondaryScreenRects = 0; let primaryScreenRects = 0; - + //render the selected frame for (let i = 0; i < frame.length; i++) { - + const a = frame[i]; for (let sI = 0; sI < a.screenLocations.length; sI++) { @@ -1238,11 +1242,18 @@ export default class Display { case 'vid': this.drawImage(a.img, screenLocation.x, screenLocation.y, a.width, a.height); break; + case 'bitmap': + this.drawImage(a.img, screenLocation.x, screenLocation.y, a.width, a.height); + break; default: continue; } primaryScreenRects++; } else { + if (!this._screens[screenLocation.screenIndex]) { + continue; + } + switch (a.type) { case 'dummy': case 'transparent': @@ -1250,7 +1261,7 @@ export default class Display { break; case 'vid': secondaryScreenRects++; - if (this._screens[screenLocation.screenIndex].channel) { + if (this._screens[screenLocation.screenIndex]?.channel) { this._screens[screenLocation.screenIndex].channel.postMessage({ eventType: 'rect', rect: { @@ -1267,6 +1278,25 @@ export default class Display { }, [a.img]); } break; + case 'bitmap': + secondaryScreenRects++; + if (this._screens[screenLocation.screenIndex].channel) { + this._screens[screenLocation.screenIndex].channel.postMessage({ + eventType: 'rect', + rect: { + 'type': 'bitmap', + 'img': a.img, + 'x': a.x, + 'y': a.y, + 'width': a.width, + 'height': a.height, + 'frame_id': a.frame_id, + 'screenLocations': a.screenLocations + }, + screenLocationIndex: sI + }, [a.img]); + } + break; case 'blit': secondaryScreenRects++; let buf = a.data.buffer; @@ -1292,7 +1322,7 @@ export default class Display { secondaryScreenRects++; if (this._screens[screenLocation.screenIndex].channel) { this._screens[screenLocation.screenIndex].channel.postMessage({ - eventType: 'rect', + eventType: 'rect', rect: { 'type': 'img', 'img': null, @@ -1327,8 +1357,8 @@ export default class Display { if (primaryScreenRects > 0) { this._writeCtxBuffer(); } - - if (this._transparentOverlayImg) { + + if (this._transparentOverlayImg) { if (primaryScreenRects > 0) { this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true); } @@ -1366,7 +1396,7 @@ export default class Display { //how many times has _pushAsyncFrame been called when the frame had all rects but has not been drawn this._asyncFrameQueue[0][5] += 1; //force the frame to be drawn if it has been here too long - if (this._asyncFrameQueue[0][5] > 5) { + if (this._asyncFrameQueue[0][5] > 5) { this._pushAsyncFrame(true); } } @@ -1384,7 +1414,7 @@ export default class Display { if ( !((rect.x > screen.x2 || screen.x > (rect.x + rect.width)) && (rect.y > screen.y2 || screen.y > (rect.y + rect.height))) ) { - let screenPosition = { + let screenPosition = { x: 0 - (screen.x - rect.x), //rect.x - screen.x, y: 0 - (screen.y - rect.y), //rect.y - screen.y, screenIndex: i @@ -1434,6 +1464,8 @@ export default class Display { this._target.style.imageRendering = 'auto'; //auto is really smooth (blurry) using trilinear of linear Log.Debug('Smoothing enabled'); } + + requestAnimationFrame( () => { this._pushAsyncFrame(); }); } _setFillColor(color) { diff --git a/core/rfb.js b/core/rfb.js index 69c2130f..d9c70ea7 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -1817,7 +1817,7 @@ export default class RFB extends EventTargetMixin { ...event.data.details, screenID: event.data.screenID } - let screenIndex = this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth, event.data.scale, event.data.serverWidth, event.data.serverHeight, event.data.x, event.data.y); + let screenIndex = this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth, event.data.scale, event.data.serverWidth, event.data.serverHeight, event.data.x, event.data.y, event.data.windowId); this._proxyRFBMessage('screenRegistrationConfirmed', [ this._display.screens[screenIndex].screenID, screenIndex ]); this._sendEncodings(); clearTimeout(this._resizeTimeout); @@ -1826,8 +1826,8 @@ export default class RFB extends EventTargetMixin { Log.Info(`Secondary monitor (${event.data.screenID}) has been registered.`); break; case 'reattach': - let changes = this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth, event.data.scale, event.data.serverWidth, event.data.serverHeight, event.data.x, event.data.y); - + let changes = this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth, event.data.scale, event.data.serverWidth, event.data.serverHeight, event.data.x, event.data.y, event.data.windowId); + clearTimeout(this._resizeTimeout); this._resizeTimeout = setTimeout(this._requestRemoteResize.bind(this), 500); this.dispatchEvent(new CustomEvent("screenregistered", {})); @@ -1971,10 +1971,12 @@ export default class RFB extends EventTargetMixin { this._display.autoscale(size.screens[0].serverWidth, size.screens[0].serverHeight, size.screens[0].scale); let screen = size.screens[0]; - + const windowId = new URLSearchParams(document.location.search).get('windowId'); + let message = { eventType: registerType, screenID: screen.screenID, + windowId, width: screen.width, height: screen.height, x: currentScreen.x || 0,