crabapples
/
noVNC
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

VNC-132 Fix image rendering logic to decrease memory usage (#141) 

* VNC-132 Fix image rendering logic to decrease memory usage

* Updating the recalculate primary display container size logic to ensure an even resolution is set

* VNC-132 Free memory after rendering bitmap image

* VNC-132 Update screen channel logic
This commit is contained in:
quickiwiki 2025-07-09 17:42:43 +05:00 committed by GitHub
parent 4d5609e9c4
commit b2a6b2a2e4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 137 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
app/ui.js
View File

@ -41,6 +41,7 @@ import Keyboard from "../core/input/keyboard.js";
import RFB from "../core/rfb.js"; import RFB from "../core/rfb.js";
import { MouseButtonMapper, XVNC_BUTTONS } from "../core/mousebuttonmapper.js"; import { MouseButtonMapper, XVNC_BUTTONS } from "../core/mousebuttonmapper.js";
import * as WebUtil from "./webutil.js"; import * as WebUtil from "./webutil.js";
import { uuidv4 } from '../core/util/strings.js';
const PAGE_TITLE = "KasmVNC"; const PAGE_TITLE = "KasmVNC";
@ -70,7 +71,8 @@ const UI = {
selectedMonitor: null, selectedMonitor: null,
refreshRotation: 0, refreshRotation: 0,
currentDisplay: null, currentDisplay: null,
displayWindows: ['primary'], displayWindows: new Map([['primary', 'primary']]),
registeredWindows: new Map([['primary', 'primary']]),
supportsBroadcastChannel: (typeof BroadcastChannel !== "undefined"), supportsBroadcastChannel: (typeof BroadcastChannel !== "undefined"),
@ -177,11 +179,11 @@ const UI = {
UI.hideKeyboardControls(); UI.hideKeyboardControls();
} }
}); });
window.addEventListener("unload", (e) => { window.addEventListener("unload", (e) => {
if (UI.rfb) { if (UI.rfb) {
UI.disconnect(); UI.disconnect();
} }
}); });
return Promise.resolve(UI.rfb); return Promise.resolve(UI.rfb);
@ -378,7 +380,7 @@ const UI = {
.pointerEvents({ holdDuration: 350 }) .pointerEvents({ holdDuration: 350 })
.on("hold", (e) => { .on("hold", (e) => {
const buttonsEl = document.querySelector(".keyboard-controls"); const buttonsEl = document.querySelector(".keyboard-controls");
const isOpen = buttonsEl.classList.contains("is-open"); const isOpen = buttonsEl.classList.contains("is-open");
buttonsEl.classList.toggle("was-open", isOpen); buttonsEl.classList.toggle("was-open", isOpen);
buttonsEl.classList.toggle("is-open", !isOpen); buttonsEl.classList.toggle("is-open", !isOpen);
@ -630,7 +632,7 @@ const UI = {
UI.addClickHandle('noVNC_identify_monitors_button', UI._identify); UI.addClickHandle('noVNC_identify_monitors_button', UI._identify);
UI.addClickHandle('noVNC_addMonitor', UI.addSecondaryMonitor); UI.addClickHandle('noVNC_addMonitor', UI.addSecondaryMonitor);
UI.addClickHandle('noVNC_refreshMonitors', UI.displaysRefresh); UI.addClickHandle('noVNC_refreshMonitors', UI.displaysRefresh);
} }
}, },
@ -657,7 +659,7 @@ const UI = {
isControlPanelItemClick(e) { isControlPanelItemClick(e) {
if (!(e && e.target && e.target.classList && e.target.parentNode && if (!(e && e.target && e.target.classList && e.target.parentNode &&
( (
e.target.classList.contains('noVNC_button') && e.target.parentNode.id !== 'noVNC_modifiers' || e.target.classList.contains('noVNC_button') && e.target.parentNode.id !== 'noVNC_modifiers' ||
e.target.classList.contains('noVNC_button_div') || e.target.classList.contains('noVNC_button_div') ||
e.target.classList.contains('noVNC_heading') e.target.classList.contains('noVNC_heading')
) )
@ -677,7 +679,7 @@ const UI = {
document.documentElement.classList.remove("noVNC_disconnected"); document.documentElement.classList.remove("noVNC_disconnected");
const transitionElem = document.getElementById("noVNC_transition_text"); const transitionElem = document.getElementById("noVNC_transition_text");
if (WebUtil.isInsideKasmVDI()) if (WebUtil.isInsideKasmVDI())
{ {
parent.postMessage({ action: 'connection_state', value: state}, '*' ); parent.postMessage({ action: 'connection_state', value: state}, '*' );
} }
@ -753,7 +755,7 @@ const UI = {
document.getElementById("noVNC_connection_stats").style.visibility = "hidden"; document.getElementById("noVNC_connection_stats").style.visibility = "hidden";
UI.statsInterval = null; UI.statsInterval = null;
} }
}, },
threading() { threading() {
@ -1459,10 +1461,10 @@ const UI = {
UI.rfb = new RFB(document.getElementById('noVNC_container'), UI.rfb = new RFB(document.getElementById('noVNC_container'),
document.getElementById('noVNC_keyboardinput'), document.getElementById('noVNC_keyboardinput'),
url, url,
{ {
shared: UI.getSetting('shared'), shared: UI.getSetting('shared'),
repeaterID: UI.getSetting('repeaterID'), repeaterID: UI.getSetting('repeaterID'),
credentials: { password: password } credentials: { password: password }
}, },
true ); true );
UI.rfb.addEventListener("connect", UI.connectFinished); UI.rfb.addEventListener("connect", UI.connectFinished);
@ -1521,9 +1523,9 @@ const UI = {
.catch(() => {}); .catch(() => {});
} }
// KASM-960 workaround, disable seamless on Safari // KASM-960 workaround, disable seamless on Safari
if (/^((?!chrome|android).)*safari/i.test(navigator.userAgent)) if (/^((?!chrome|android).)*safari/i.test(navigator.userAgent))
{ {
UI.rfb.clipboardSeamless = false; UI.rfb.clipboardSeamless = false;
} }
UI.rfb.preferLocalCursor = UI.getSetting('prefer_local_cursor'); UI.rfb.preferLocalCursor = UI.getSetting('prefer_local_cursor');
UI.rfb.enableWebP = UI.getSetting('enable_webp'); UI.rfb.enableWebP = UI.getSetting('enable_webp');
@ -1540,7 +1542,7 @@ const UI = {
window.attachEvent('onload', WindowLoad); window.attachEvent('onload', WindowLoad);
window.attachEvent('message', UI.receiveMessage); window.attachEvent('message', UI.receiveMessage);
} }
if (UI.rfb.clipboardDown){ if (UI.rfb.clipboardDown){
UI.rfb.addEventListener("clipboard", UI.clipboardRx); UI.rfb.addEventListener("clipboard", UI.clipboardRx);
} }
UI.rfb.addEventListener("disconnect", UI.disconnectedRx); UI.rfb.addEventListener("disconnect", UI.disconnectedRx);
@ -1553,7 +1555,7 @@ const UI = {
UI._sessionTimeoutInterval = setInterval(function() { UI._sessionTimeoutInterval = setInterval(function() {
if (UI.rfb) { if (UI.rfb) {
const timeSinceLastActivityInS = (Date.now() - UI.rfb.lastActiveAt) / 1000; const timeSinceLastActivityInS = (Date.now() - UI.rfb.lastActiveAt) / 1000;
let idleDisconnectInS = 1200; //20 minute default let idleDisconnectInS = 1200; //20 minute default
if (Number.isFinite(parseFloat(UI.rfb.idleDisconnect))) { if (Number.isFinite(parseFloat(UI.rfb.idleDisconnect))) {
idleDisconnectInS = parseFloat(UI.rfb.idleDisconnect) * 60; idleDisconnectInS = parseFloat(UI.rfb.idleDisconnect) * 60;
} }
@ -2009,8 +2011,9 @@ const UI = {
}, },
async addSecondaryMonitor() { async addSecondaryMonitor() {
let new_display_path = window.location.pathname.replace(/[^/]*$/, '') let new_display_path = window.location.pathname.replace(/[^/]*$/, '');
let new_display_url = `${window.location.protocol}//${window.location.host}${new_display_path}screen.html`; const windowId = uuidv4();
let new_display_url = `${window.location.protocol}//${window.location.host}${new_display_path}screen.html?windowId=${windowId}`;
const auto_placement = document.getElementById('noVNC_auto_placement').checked const auto_placement = document.getElementById('noVNC_auto_placement').checked
if (auto_placement && 'getScreenDetails' in window) { if (auto_placement && 'getScreenDetails' in window) {
@ -2020,11 +2023,11 @@ const UI = {
permission = (state === 'granted' || state === 'prompt'); permission = (state === 'granted' || state === 'prompt');
if (permission && window.screen.isExtended) { if (permission && window.screen.isExtended) {
const details = await window.getScreenDetails() const details = await window.getScreenDetails()
const current = UI.increaseCurrentDisplay(details) const current = UI.increaseCurrentDisplay(details)
let screen = details.screens[current] let screen = details.screens[current]
const options = 'left='+screen.availLeft+',top='+screen.availTop+',width='+screen.availWidth+',height='+screen.availHeight+',fullscreen' const options = 'left='+screen.availLeft+',top='+screen.availTop+',width='+screen.availWidth+',height='+screen.availHeight+',fullscreen'
let newdisplay = window.open(new_display_url, '_blank', options); let newdisplay = window.open(new_display_url, '_blank', options);
UI.displayWindows.push(newdisplay); UI.displayWindows.set(windowId, newdisplay);
return; return;
} }
} catch (e) { } catch (e) {
@ -2032,17 +2035,19 @@ const UI = {
// Nothing. // Nothing.
} }
} }
Log.Debug(`Opening a secondary display ${new_display_url}`) Log.Debug(`Opening a secondary display ${new_display_url}`)
let newdisplay = window.open(new_display_url, '_blank', 'toolbar=0,location=0,menubar=0'); let newdisplay = window.open(new_display_url, '_blank', 'toolbar=0,location=0,menubar=0');
UI.displayWindows.push(newdisplay); if (newdisplay) {
UI.displayWindows.set(windowId, newdisplay);
}
}, },
initMonitors(screenPlan) { initMonitors(screenPlan) {
const { scale } = UI.multiMonitorSettings() const { scale } = UI.multiMonitorSettings()
let monitors = [] let monitors = []
let showNativeResolution = false let showNativeResolution = false
let num = 1 let num = 1;
screenPlan.screens.forEach(screen => { screenPlan.screens.forEach(screen => {
if (parseFloat(screen.pixelRatio) != 1) { if (parseFloat(screen.pixelRatio) != 1) {
showNativeResolution = true showNativeResolution = true
@ -2078,7 +2083,7 @@ const UI = {
}, },
updateMonitors(screenPlan) { updateMonitors(screenPlan) {
UI.initMonitors(screenPlan) UI.initMonitors(screenPlan)
UI.recenter() UI.recenter()
UI.draw() UI.draw()
}, },
@ -2132,7 +2137,7 @@ const UI = {
prev = monitors[i] prev = monitors[i]
} }
} }
}, },
rect(ctx, x, y, w, h) { rect(ctx, x, y, w, h) {
ctx.beginPath(); ctx.beginPath();
@ -2223,7 +2228,9 @@ const UI = {
serverWidth: Math.round(width * scale), serverWidth: Math.round(width * scale),
screens screens
} }
UI.rfb.applyScreenPlan(screenPlan); if (UI.rfb) {
UI.rfb.applyScreenPlan(screenPlan);
}
}, },
@ -2238,7 +2245,7 @@ const UI = {
let dragok = false let dragok = false
let startX; let startX;
let startY; let startY;
offsetX = bb.left offsetX = bb.left
offsetY = bb.top offsetY = bb.top
@ -2306,7 +2313,7 @@ const UI = {
var dx = mx - startX; var dx = mx - startX;
var dy = my - startY; var dy = my - startY;
// move each rect that isDragging // move each rect that isDragging
// by the distance the mouse has moved // by the distance the mouse has moved
// since the last mousemove // since the last mousemove
for (var i = 0; i < monitors.length; i++) { for (var i = 0; i < monitors.length; i++) {
@ -2910,7 +2917,7 @@ const UI = {
UI.showControlInput("noVNC_keyboard_button"); UI.showControlInput("noVNC_keyboard_button");
UI.showControlInput("noVNC_toggle_extra_keys_button"); UI.showControlInput("noVNC_toggle_extra_keys_button");
UI.showControlInput("noVNC_clipboard_button"); UI.showControlInput("noVNC_clipboard_button");
UI.showControlInput("noVNC_game_mode_button"); UI.showControlInput("noVNC_game_mode_button");
} }
}, },
@ -2939,7 +2946,7 @@ const UI = {
UI.closeControlbar(); UI.closeControlbar();
UI.showStatus('Press Esc Key to Exit Pointer Lock Mode', 'warn', 5000, true); UI.showStatus('Press Esc Key to Exit Pointer Lock Mode', 'warn', 5000, true);
} else { } else {
//If in game mode //If in game mode
if (UI.rfb.pointerRelative) { if (UI.rfb.pointerRelative) {
UI.showStatus('Game Mode paused, click on screen to resume Game Mode.', 'warn', 5000, true); UI.showStatus('Game Mode paused, click on screen to resume Game Mode.', 'warn', 5000, true);
} else { } else {
@ -2982,7 +2989,7 @@ const UI = {
screenRegistered(e) { screenRegistered(e) {
console.log('screen registered') console.log('screen registered')
// Get the current screen plan // Get the current screen plan
// When a new display is added, it is defaulted to be placed to the far right relative to existing displays and to the top // When a new display is added, it is defaulted to be placed to the far right relative to existing displays and to the top
if (UI.rfb) { if (UI.rfb) {
@ -2999,7 +3006,7 @@ const UI = {
UI.updateMonitors(screenPlan) UI.updateMonitors(screenPlan)
UI._identify(UI.monitors) UI._identify(UI.monitors)
} }
}, },
//Helper to add options to dropdown. //Helper to add options to dropdown.

148
core/display.js
View File

@ -100,7 +100,7 @@ export default class Display {
this._fps = 0; this._fps = 0;
this._isPrimaryDisplay = isPrimaryDisplay; this._isPrimaryDisplay = isPrimaryDisplay;
this._screenID = uuidv4(); this._screenID = uuidv4();
this._screens = [{ this._screens = [{
screenID: this._screenID, screenID: this._screenID,
screenIndex: 0, screenIndex: 0,
width: this._target.width, //client width: this._target.width, //client
@ -150,7 +150,7 @@ export default class Display {
this._enableCanvasBuffer = value; this._enableCanvasBuffer = value;
if (value && this._target) if (value && this._target)
{ {
//copy current visible canvas to backbuffer //copy current visible canvas to backbuffer
@ -175,9 +175,9 @@ export default class Display {
if (!this._isPrimaryDisplay && this._screens[0].screenIndex == 0) { if (!this._isPrimaryDisplay && this._screens[0].screenIndex == 0) {
return -1; return -1;
} }
return this._screens[0].screenIndex; return this._screens[0].screenIndex;
} }
get antiAliasing() { return this._antiAliasing; } get antiAliasing() { return this._antiAliasing; }
set antiAliasing(value) { set antiAliasing(value) {
this._antiAliasing = value; this._antiAliasing = value;
@ -228,7 +228,7 @@ export default class Display {
*/ */
getClientRelativeCoordinates(x, y) { getClientRelativeCoordinates(x, y) {
for (let i = 0; i < this._screens.length; i++) { for (let i = 0; i < this._screens.length; i++) {
if ( if (
(x >= this._screens[i].x && x <= this._screens[i].x + this._screens[i].serverWidth) && (x >= this._screens[i].x && x <= this._screens[i].x + this._screens[i].serverWidth) &&
(y >= this._screens[i].y && y <= this._screens[i].y + this._screens[i].serverHeight) (y >= this._screens[i].y && y <= this._screens[i].y + this._screens[i].serverHeight)
) )
@ -242,7 +242,7 @@ export default class Display {
} }
} }
/* /*
Returns coordinates that are server relative when multiple monitors are in use Returns coordinates that are server relative when multiple monitors are in use
*/ */
getServerRelativeCoordinates(screenIndex, x, y) { getServerRelativeCoordinates(screenIndex, x, y) {
@ -263,9 +263,9 @@ export default class Display {
let i = 0; let i = 0;
//getting parent node size with sub-pixel precision //getting parent node size with sub-pixel precision
let parentNodeSize = this._target.parentNode.getBoundingClientRect(); let parentNodeSize = this._target.parentNode.getBoundingClientRect();
//recalculate primary display container size //recalculate primary display container size
this._screens[i].containerHeight = Math.floor(parentNodeSize.height / 2) * 2; this._screens[i].containerHeight = Math.floor(parentNodeSize.height / 2) * 2;
this._screens[i].containerWidth = Math.floor(parentNodeSize.width / 2) * 2; this._screens[i].containerWidth = Math.floor(parentNodeSize.width / 2) * 2;
@ -284,7 +284,7 @@ export default class Display {
( (
disableScaling || disableScaling ||
(this._screens[i].serverReportedWidth !== this._screens[i].serverWidth || this._screens[i].serverReportedHeight !== this._screens[i].serverHeight) (this._screens[i].serverReportedWidth !== this._screens[i].serverWidth || this._screens[i].serverReportedHeight !== this._screens[i].serverHeight)
) && ) &&
(!max_width && !max_height) (!max_width && !max_height)
) { ) {
height = this._screens[i].serverReportedHeight; height = this._screens[i].serverReportedHeight;
@ -307,7 +307,7 @@ export default class Display {
} }
//physically small device with high DPI //physically small device with high DPI
else if (this._antiAliasing === 0 && this._screens[i].pixelRatio > 1 && width < 1000 & width > 0) { else if (this._antiAliasing === 0 && this._screens[i].pixelRatio > 1 && width < 1000 & width > 0) {
Log.Info('Device Pixel ratio: ' + this._screens[i].pixelRatio + ' Reported Resolution: ' + width + 'x' + height); Log.Info('Device Pixel ratio: ' + this._screens[i].pixelRatio + ' Reported Resolution: ' + width + 'x' + height);
let targetDevicePixelRatio = 1.5; let targetDevicePixelRatio = 1.5;
if (this._screens[i].pixelRatio > 2) { targetDevicePixelRatio = 2; } if (this._screens[i].pixelRatio > 2) { targetDevicePixelRatio = 2; }
let scaledWidth = (width * this._screens[i].pixelRatio) * (1 / targetDevicePixelRatio); let scaledWidth = (width * this._screens[i].pixelRatio) * (1 / targetDevicePixelRatio);
@ -317,10 +317,10 @@ export default class Display {
scale = 1 / scaleRatio; scale = 1 / scaleRatio;
Log.Info('Small device with hDPI screen detected, auto scaling at ' + scaleRatio + ' to ' + width + 'x' + height); Log.Info('Small device with hDPI screen detected, auto scaling at ' + scaleRatio + ' to ' + width + 'x' + height);
} }
let clientServerRatioH = this._screens[i].containerHeight / height; let clientServerRatioH = this._screens[i].containerHeight / height;
let clientServerRatioW = this._screens[i].containerWidth / width; let clientServerRatioW = this._screens[i].containerWidth / width;
this._screens[i].height = Math.floor(height * clientServerRatioH); this._screens[i].height = Math.floor(height * clientServerRatioH);
this._screens[i].width = Math.floor(width * clientServerRatioW); this._screens[i].width = Math.floor(width * clientServerRatioW);
this._screens[i].serverWidth = width; this._screens[i].serverWidth = width;
@ -376,12 +376,12 @@ export default class Display {
return changes; return changes;
} }
addScreen(screenID, width, height, pixelRatio, containerHeight, containerWidth, scale, serverWidth, serverHeight, x, y) { addScreen(screenID, width, height, pixelRatio, containerHeight, containerWidth, scale, serverWidth, serverHeight, x, y, windowId) {
if (!this._isPrimaryDisplay) { if (!this._isPrimaryDisplay) {
throw new Error("Cannot add a screen to a secondary display."); throw new Error("Cannot add a screen to a secondary display.");
} }
else if (containerHeight === 0 || containerWidth === 0 || pixelRatio === 0) { else if (containerHeight === 0 || containerWidth === 0 || pixelRatio === 0) {
Log.Warn("Invalid screen configuration."); Log.Warn("Invalid screen configuration.");
} }
let screenIdx = -1; let screenIdx = -1;
@ -395,8 +395,8 @@ export default class Display {
if (screenIdx > 0) { if (screenIdx > 0) {
//existing screen, update //existing screen, update
const existing_screen = this._screens[screenIdx]; const existing_screen = this._screens[screenIdx];
if (existing_screen.serverHeight !== serverHeight || existing_screen.serverWidth !== serverWidth || existing_screen.width !== width || existing_screen.height !== height if (existing_screen.serverHeight !== serverHeight || existing_screen.serverWidth !== serverWidth || existing_screen.width !== width || existing_screen.height !== height
|| existing_screen.containerHeight !== containerHeight || existing_screen.containerWidth !== containerWidth || existing_screen.scale !== scale || existing_screen.pixelRatio !== pixelRatio || || existing_screen.containerHeight !== containerHeight || existing_screen.containerWidth !== containerWidth || existing_screen.scale !== scale || existing_screen.pixelRatio !== pixelRatio ||
existing_screen.x !== x || existing_screen.y !== y) { existing_screen.x !== x || existing_screen.y !== y) {
existing_screen.width = width; existing_screen.width = width;
existing_screen.height = height; existing_screen.height = height;
@ -432,14 +432,18 @@ export default class Display {
pixelRatio: pixelRatio, pixelRatio: pixelRatio,
containerHeight: containerHeight, containerHeight: containerHeight,
containerWidth: containerWidth, containerWidth: containerWidth,
channel: UI.displayWindows[this.screens.length], channel: UI.displayWindows.get(windowId),
scale: scale, scale: scale,
x2: x + serverWidth, x2: x + serverWidth,
y2: serverHeight y2: serverHeight
} }
this._screens.push(new_screen); this._screens.push(new_screen);
new_screen.channel.postMessage({ eventType: "registered", screenIndex: new_screen.screenIndex }); if (new_screen.channel) {
UI.registeredWindows.set(screenID, windowId);
new_screen.channel.postMessage({eventType: "registered", screenIndex: new_screen.screenIndex});
} else
Log.Debug(`Channel not found for screenId ${screenID}`);
return new_screen.screenIndex; return new_screen.screenIndex;
} }
@ -454,7 +458,11 @@ export default class Display {
if (this._screens[i].screenID == screenID) { if (this._screens[i].screenID == screenID) {
//flush all rects on target screen //flush all rects on target screen
this._flushRectsScreen(i); this._flushRectsScreen(i);
UI.displayWindows.splice(i, 1); const windowId = UI.registeredWindows.get(screenID);
if (windowId) {
UI.registeredWindows.delete(screenID);
UI.displayWindows.delete(windowId);
}
this._screens.splice(i, 1); this._screens.splice(i, 1);
removed = true; removed = true;
break; break;
@ -577,7 +585,7 @@ export default class Display {
let canvas = this._backbuffer; let canvas = this._backbuffer;
if (canvas == undefined) { return; } if (canvas == undefined) { return; }
if (this._screens.length > 0) { if (this._screens.length > 0) {
width = this._screens[0].serverWidth; width = this._screens[0].serverWidth;
height = this._screens[0].serverHeight; height = this._screens[0].serverHeight;
@ -603,7 +611,7 @@ export default class Display {
} }
} }
// Readjust the viewport as it may be incorrectly sized // Readjust the viewport as it may be incorrectly sized
// and positioned // and positioned
@ -648,7 +656,7 @@ export default class Display {
if (onflush_message) if (onflush_message)
this.onflush(); this.onflush();
} }
/* /*
* Clears the buffer of anything that has not yet been displayed. * Clears the buffer of anything that has not yet been displayed.
* This must be called when switching between transit modes tcp/udp * This must be called when switching between transit modes tcp/udp
@ -743,7 +751,7 @@ export default class Display {
if ((typeof ImageDecoder !== 'undefined') && (this._threading)) { if ((typeof ImageDecoder !== 'undefined') && (this._threading)) {
let imageDecoder = new ImageDecoder({ data: arr, type: mime }); let imageDecoder = new ImageDecoder({ data: arr, type: mime });
let rect = { let rect = {
'type': 'vid', 'type': 'vid',
'img': null, 'img': null,
'x': x, 'x': x,
'y': y, 'y': y,
@ -757,30 +765,22 @@ export default class Display {
return; return;
} }
let rect = { const blob = new Blob([arr], { type: mime });
'type': 'img', const rect = {
'type': 'bitmap',
'img': null, 'img': null,
'x': x, 'x': x,
'y': y, 'y': y,
'width': width, 'width': width,
'height': height, 'height': height,
'frame_id': frame_id 'frame_id': frame_id,
} 'mime': mime
};
this._processRectScreens(rect); this._processRectScreens(rect);
createImageBitmap(blob).then((bitmapImg) => {
if (rect.inPrimary) { rect.img = bitmapImg;
const img = new Image(); this._asyncRenderQPush(rect);
img.src = "data: " + mime + ";base64," + Base64.encode(arr); });
rect.img = img;
} else {
rect.type = "_img";
}
if (rect.inSecondary) {
rect.mime = mime;
rect.src = "data: " + mime + ";base64," + Base64.encode(arr);
}
this._asyncRenderQPush(rect);
} }
transparentRect(x, y, width, height, img, frame_id, hashId) { transparentRect(x, y, width, height, img, frame_id, hashId) {
@ -832,8 +832,8 @@ export default class Display {
if (!fromQueue) { if (!fromQueue) {
var buf; var buf;
if (!ArrayBuffer.isView(arr)) { if (!ArrayBuffer.isView(arr)) {
buf = arr; buf = arr;
} else { } else {
buf = arr.buffer; buf = arr.buffer;
} }
// NB(directxman12): it's technically more performant here to use preallocated arrays, // NB(directxman12): it's technically more performant here to use preallocated arrays,
@ -869,9 +869,9 @@ export default class Display {
this._drawCtx.putImageData(img, x, y); this._drawCtx.putImageData(img, x, y);
} else { } else {
this._targetCtx.putImageData(img, x, y); this._targetCtx.putImageData(img, x, y);
} }
} }
} }
@ -1027,6 +1027,10 @@ export default class Display {
this.drawImage(a.img, pos.x, pos.y, a.width, a.height); this.drawImage(a.img, pos.x, pos.y, a.width, a.height);
a.img.close(); a.img.close();
break; break;
case 'bitmap':
this.drawImage(a.img, pos.x, pos.y, a.width, a.height);
a.img.close();
break;
default: default:
this._syncFrameQueue.shift(); this._syncFrameQueue.shift();
continue; continue;
@ -1100,7 +1104,7 @@ export default class Display {
this._asyncFrameQueue[frameIx][1] += rect.rect_cnt; this._asyncFrameQueue[frameIx][1] += rect.rect_cnt;
if (rect.rect_cnt == 0) { if (rect.rect_cnt == 0) {
Log.Warn("Invalid rect count"); Log.Warn("Invalid rect count");
} }
} }
if (this._asyncFrameQueue[frameIx][1] > 0 && this._asyncFrameQueue[frameIx][2].length >= this._asyncFrameQueue[frameIx][1]) { if (this._asyncFrameQueue[frameIx][1] > 0 && this._asyncFrameQueue[frameIx][2].length >= this._asyncFrameQueue[frameIx][1]) {
@ -1125,13 +1129,13 @@ export default class Display {
this._asyncFrameQueue.shift(); this._asyncFrameQueue.shift();
this._droppedFrames += (rect.frame_id - newestFrameID); this._droppedFrames += (rect.frame_id - newestFrameID);
} }
let rect_cnt = ((rect.type == "flip") ? rect.rect_cnt : 0); let rect_cnt = ((rect.type == "flip") ? rect.rect_cnt : 0);
this._asyncFrameQueue.push([ rect.frame_id, rect_cnt, [ rect ], (rect_cnt == 1), 0, 0 ]); this._asyncFrameQueue.push([ rect.frame_id, rect_cnt, [ rect ], (rect_cnt == 1), 0, 0 ]);
} }
} }
} }
/* /*
@ -1166,7 +1170,7 @@ export default class Display {
Log.Warn("Frame has more rects than the reported rect_cnt."); Log.Warn("Frame has more rects than the reported rect_cnt.");
} }
} }
while (currentFrameRectIx < this._asyncFrameQueue[frameIx][2].length) { while (currentFrameRectIx < this._asyncFrameQueue[frameIx][2].length) {
if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img') { if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type == 'img') {
if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) { if (this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img && !this._asyncFrameQueue[frameIx][2][currentFrameRectIx].img.complete) {
this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type = 'skip'; this._asyncFrameQueue[frameIx][2][currentFrameRectIx].type = 'skip';
@ -1210,10 +1214,10 @@ export default class Display {
let secondaryScreenRects = 0; let secondaryScreenRects = 0;
let primaryScreenRects = 0; let primaryScreenRects = 0;
//render the selected frame //render the selected frame
for (let i = 0; i < frame.length; i++) { for (let i = 0; i < frame.length; i++) {
const a = frame[i]; const a = frame[i];
for (let sI = 0; sI < a.screenLocations.length; sI++) { for (let sI = 0; sI < a.screenLocations.length; sI++) {
@ -1238,11 +1242,18 @@ export default class Display {
case 'vid': case 'vid':
this.drawImage(a.img, screenLocation.x, screenLocation.y, a.width, a.height); this.drawImage(a.img, screenLocation.x, screenLocation.y, a.width, a.height);
break; break;
case 'bitmap':
this.drawImage(a.img, screenLocation.x, screenLocation.y, a.width, a.height);
break;
default: default:
continue; continue;
} }
primaryScreenRects++; primaryScreenRects++;
} else { } else {
if (!this._screens[screenLocation.screenIndex]) {
continue;
}
switch (a.type) { switch (a.type) {
case 'dummy': case 'dummy':
case 'transparent': case 'transparent':
@ -1250,7 +1261,7 @@ export default class Display {
break; break;
case 'vid': case 'vid':
secondaryScreenRects++; secondaryScreenRects++;
if (this._screens[screenLocation.screenIndex].channel) { if (this._screens[screenLocation.screenIndex]?.channel) {
this._screens[screenLocation.screenIndex].channel.postMessage({ this._screens[screenLocation.screenIndex].channel.postMessage({
eventType: 'rect', eventType: 'rect',
rect: { rect: {
@ -1267,6 +1278,25 @@ export default class Display {
}, [a.img]); }, [a.img]);
} }
break; break;
case 'bitmap':
secondaryScreenRects++;
if (this._screens[screenLocation.screenIndex].channel) {
this._screens[screenLocation.screenIndex].channel.postMessage({
eventType: 'rect',
rect: {
'type': 'bitmap',
'img': a.img,
'x': a.x,
'y': a.y,
'width': a.width,
'height': a.height,
'frame_id': a.frame_id,
'screenLocations': a.screenLocations
},
screenLocationIndex: sI
}, [a.img]);
}
break;
case 'blit': case 'blit':
secondaryScreenRects++; secondaryScreenRects++;
let buf = a.data.buffer; let buf = a.data.buffer;
@ -1292,7 +1322,7 @@ export default class Display {
secondaryScreenRects++; secondaryScreenRects++;
if (this._screens[screenLocation.screenIndex].channel) { if (this._screens[screenLocation.screenIndex].channel) {
this._screens[screenLocation.screenIndex].channel.postMessage({ this._screens[screenLocation.screenIndex].channel.postMessage({
eventType: 'rect', eventType: 'rect',
rect: { rect: {
'type': 'img', 'type': 'img',
'img': null, 'img': null,
@ -1327,8 +1357,8 @@ export default class Display {
if (primaryScreenRects > 0) { if (primaryScreenRects > 0) {
this._writeCtxBuffer(); this._writeCtxBuffer();
} }
if (this._transparentOverlayImg) { if (this._transparentOverlayImg) {
if (primaryScreenRects > 0) { if (primaryScreenRects > 0) {
this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true); this.drawImage(this._transparentOverlayImg, this._transparentOverlayRect.x, this._transparentOverlayRect.y, this._transparentOverlayRect.width, this._transparentOverlayRect.height, true);
} }
@ -1366,7 +1396,7 @@ export default class Display {
//how many times has _pushAsyncFrame been called when the frame had all rects but has not been drawn //how many times has _pushAsyncFrame been called when the frame had all rects but has not been drawn
this._asyncFrameQueue[0][5] += 1; this._asyncFrameQueue[0][5] += 1;
//force the frame to be drawn if it has been here too long //force the frame to be drawn if it has been here too long
if (this._asyncFrameQueue[0][5] > 5) { if (this._asyncFrameQueue[0][5] > 5) {
this._pushAsyncFrame(true); this._pushAsyncFrame(true);
} }
} }
@ -1384,7 +1414,7 @@ export default class Display {
if ( if (
!((rect.x > screen.x2 || screen.x > (rect.x + rect.width)) && (rect.y > screen.y2 || screen.y > (rect.y + rect.height))) !((rect.x > screen.x2 || screen.x > (rect.x + rect.width)) && (rect.y > screen.y2 || screen.y > (rect.y + rect.height)))
) { ) {
let screenPosition = { let screenPosition = {
x: 0 - (screen.x - rect.x), //rect.x - screen.x, x: 0 - (screen.x - rect.x), //rect.x - screen.x,
y: 0 - (screen.y - rect.y), //rect.y - screen.y, y: 0 - (screen.y - rect.y), //rect.y - screen.y,
screenIndex: i screenIndex: i
@ -1434,6 +1464,8 @@ export default class Display {
this._target.style.imageRendering = 'auto'; //auto is really smooth (blurry) using trilinear of linear this._target.style.imageRendering = 'auto'; //auto is really smooth (blurry) using trilinear of linear
Log.Debug('Smoothing enabled'); Log.Debug('Smoothing enabled');
} }
requestAnimationFrame( () => { this._pushAsyncFrame(); });
} }
_setFillColor(color) { _setFillColor(color) {

10
core/rfb.js
View File

@ -1817,7 +1817,7 @@ export default class RFB extends EventTargetMixin {
...event.data.details, ...event.data.details,
screenID: event.data.screenID screenID: event.data.screenID
} }
let screenIndex = this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth, event.data.scale, event.data.serverWidth, event.data.serverHeight, event.data.x, event.data.y); let screenIndex = this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth, event.data.scale, event.data.serverWidth, event.data.serverHeight, event.data.x, event.data.y, event.data.windowId);
this._proxyRFBMessage('screenRegistrationConfirmed', [ this._display.screens[screenIndex].screenID, screenIndex ]); this._proxyRFBMessage('screenRegistrationConfirmed', [ this._display.screens[screenIndex].screenID, screenIndex ]);
this._sendEncodings(); this._sendEncodings();
clearTimeout(this._resizeTimeout); clearTimeout(this._resizeTimeout);
@ -1826,8 +1826,8 @@ export default class RFB extends EventTargetMixin {
Log.Info(`Secondary monitor (${event.data.screenID}) has been registered.`); Log.Info(`Secondary monitor (${event.data.screenID}) has been registered.`);
break; break;
case 'reattach': case 'reattach':
let changes = this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth, event.data.scale, event.data.serverWidth, event.data.serverHeight, event.data.x, event.data.y); let changes = this._display.addScreen(event.data.screenID, event.data.width, event.data.height, event.data.pixelRatio, event.data.containerHeight, event.data.containerWidth, event.data.scale, event.data.serverWidth, event.data.serverHeight, event.data.x, event.data.y, event.data.windowId);
clearTimeout(this._resizeTimeout); clearTimeout(this._resizeTimeout);
this._resizeTimeout = setTimeout(this._requestRemoteResize.bind(this), 500); this._resizeTimeout = setTimeout(this._requestRemoteResize.bind(this), 500);
this.dispatchEvent(new CustomEvent("screenregistered", {})); this.dispatchEvent(new CustomEvent("screenregistered", {}));
@ -1971,10 +1971,12 @@ export default class RFB extends EventTargetMixin {
this._display.autoscale(size.screens[0].serverWidth, size.screens[0].serverHeight, size.screens[0].scale); this._display.autoscale(size.screens[0].serverWidth, size.screens[0].serverHeight, size.screens[0].scale);
let screen = size.screens[0]; let screen = size.screens[0];
const windowId = new URLSearchParams(document.location.search).get('windowId');
let message = { let message = {
eventType: registerType, eventType: registerType,
screenID: screen.screenID, screenID: screen.screenID,
windowId,
width: screen.width, width: screen.width,
height: screen.height, height: screen.height,
x: currentScreen.x || 0, x: currentScreen.x || 0,