From 6da3b14d025fce1f7c3ba942d9ed76657651f479 Mon Sep 17 00:00:00 2001 From: Matt McClaskey Date: Thu, 17 Feb 2022 10:57:08 -0500 Subject: [PATCH] WIP Added udp decoder --- app/ui.js | 4 +- core/decoders/udp.js | 299 +++++++++++++++++++++++++++++++++++++++++++ core/encodings.js | 1 + core/rfb.js | 137 +++++++++++--------- vnc.html | 4 +- 5 files changed, 378 insertions(+), 67 deletions(-) create mode 100644 core/decoders/udp.js diff --git a/app/ui.js b/app/ui.js index 3d5bb0f0..bc539971 100644 --- a/app/ui.js +++ b/app/ui.js @@ -30,8 +30,8 @@ window.updateSetting = (name, value) => { } } -import "core-js/stable"; -import "regenerator-runtime/runtime"; +//import "core-js/stable"; +//import "regenerator-runtime/runtime"; import * as Log from '../core/util/logging.js'; import _, { l10n } from './localization.js'; import { isTouchDevice, isSafari, hasScrollbarGutter, dragThreshold, supportsBinaryClipboard, isFirefox, isWindows, isIOS } diff --git a/core/decoders/udp.js b/core/decoders/udp.js new file mode 100644 index 00000000..75d11ba0 --- /dev/null +++ b/core/decoders/udp.js @@ -0,0 +1,299 @@ +/* + * noVNC: HTML5 VNC client + * Copyright (C) 2019 The noVNC Authors + * (c) 2012 Michael Tinglof, Joe Balaz, Les Piech (Mercuri.ca) + * Licensed under MPL 2.0 (see LICENSE.txt) + * + * See README.md for usage and integration instructions. + * + */ + +import * as Log from '../util/logging.js'; +import Inflator from "../inflator.js"; + +export default class UDPDecoder { + constructor() { + this._filter = null; + this._numColors = 0; + this._palette = new Uint8Array(1024); // 256 * 4 (max palette size * max bytes-per-pixel) + + this._zlibs = []; + for (let i = 0; i < 4; i++) { + this._zlibs[i] = new Inflator(); + } + } + + decodeRect(x, y, width, height, data, display, depth) { + let ctl = data[12]; + ctl = ctl >> 4; + + let ret; + + if (ctl === 0x08) { + Log.Info("Fill Rect"); + ret = this._fillRect(x, y, width, height, + data, display, depth); + } else if (ctl === 0x09) { + Log.Info("Fill JPEG"); + ret = this._jpegRect(x, y, width, height, + data, display, depth); + } else if (ctl === 0x0A) { + Log.Info("Fill Png"); + ret = this._pngRect(x, y, width, height, + data, display, depth); + } else if ((ctl & 0x08) == 0) { + Log.Info("Fill Basic"); + ret = this._basicRect(ctl, x, y, width, height, + data, display, depth); + } else if (ctl === 0x0B) { + Log.Info("Fill webp"); + ret = this._webpRect(x, y, width, height, + data, display, depth); + } else { + throw new Error("Illegal udp compression received (ctl: " + + ctl + ")"); + } + + return ret; + } + + _fillRect(x, y, width, height, data, display, depth) { + + display.fillRect(x, y, width, height, + [data[13], data[14], data[15]], false); + + return true; + } + + _jpegRect(x, y, width, height, data, display, depth) { + let img = this._readData(data); + if (img === null) { + return false; + } + + display.imageRect(x, y, width, height, "image/jpeg", img); + + return true; + } + + _webpRect(x, y, width, height, data, display, depth) { + let img = this._readData(data); + if (img === null) { + return false; + } + + display.imageRect(x, y, width, height, "image/webp", img); + + return true; + } + + _pngRect(x, y, width, height, data, display, depth) { + //throw new Error("PNG received in UDP rect"); + Log.Error("PNG received in UDP rect"); + } + + _basicRect(ctl, x, y, width, height, data, display, depth) { + let zlibs_flags = data[12]; + // Reset streams if the server requests it + for (let i = 0; i < 4; i++) { + if ((zlibs_flags >> i) & 1) { + this._zlibs[i].reset(); + Log.Info("Reset zlib stream " + i); + } + } + + let filter = data[13]; + let data_index = 14; + let streamId = ctl & 0x3; + if (!(ctl & 0x4)) { + // Implicit CopyFilter + filter = 0; + data_index = 13; + } + + let ret; + + switch (filter) { + case 0: // CopyFilter + Log.Info("Filter Copy"); + ret = this._copyFilter(streamId, x, y, width, height, + data, display, depth, data_index); + break; + case 1: // PaletteFilter + Log.Info("Filter Palette"); + ret = this._paletteFilter(streamId, x, y, width, height, + data, display, depth); + break; + case 2: // GradientFilter + Log.Info("Filter Gradient"); + ret = this._gradientFilter(streamId, x, y, width, height, + data, display, depth); + break; + default: + throw new Error("Illegal tight filter received (ctl: " + + this._filter + ")"); + } + + return ret; + } + + _copyFilter(streamId, x, y, width, height, data, display, depth, data_index=14) { + const uncompressedSize = width * height * 3; + + if (uncompressedSize === 0) { + return true; + } + + if (uncompressedSize < 12) { + data = data.slice(data_index, data_index + uncompressedSize); + //data = sock.rQshiftBytes(uncompressedSize); + } else { + data = this._readData(data, data_index); + if (data === null) { + return false; + } + + this._zlibs[streamId].setInput(data); + data = this._zlibs[streamId].inflate(uncompressedSize); + this._zlibs[streamId].setInput(null); + } + + let rgbx = new Uint8Array(width * height * 4); + for (let i = 0, j = 0; i < width * height * 4; i += 4, j += 3) { + rgbx[i] = data[j]; + rgbx[i + 1] = data[j + 1]; + rgbx[i + 2] = data[j + 2]; + rgbx[i + 3] = 255; // Alpha + } + + display.blitImage(x, y, width, height, rgbx, 0, false); + + return true; + } + + _paletteFilter(streamId, x, y, width, height, data, display, depth) { + const numColors = data[14] + 1; + const paletteSize = numColors * 3; + let palette = data.slice(15, 15 + paletteSize); + + const bpp = (numColors <= 2) ? 1 : 8; + const rowSize = Math.floor((width * bpp + 7) / 8); + const uncompressedSize = rowSize * height; + let data_i = 15 + paletteSize; + + if (uncompressedSize === 0) { + return true; + } + + if (uncompressedSize < 12) { + //data = sock.rQshiftBytes(uncompressedSize); + data = data.slice(data_i, data_i + uncompressedSize); + } else { + data = this._readData(data, data_i); + if (data === null) { + return false; + } + + this._zlibs[streamId].setInput(data); + data = this._zlibs[streamId].inflate(uncompressedSize); + this._zlibs[streamId].setInput(null); + } + + // Convert indexed (palette based) image data to RGB + if (this._numColors == 2) { + this._monoRect(x, y, width, height, data, palette, display); + } else { + this._paletteRect(x, y, width, height, data, palette, display); + } + + return true; + } + + _monoRect(x, y, width, height, data, palette, display) { + // Convert indexed (palette based) image data to RGB + // TODO: reduce number of calculations inside loop + const dest = this._getScratchBuffer(width * height * 4); + const w = Math.floor((width + 7) / 8); + const w1 = Math.floor(width / 8); + + for (let y = 0; y < height; y++) { + let dp, sp, x; + for (x = 0; x < w1; x++) { + for (let b = 7; b >= 0; b--) { + dp = (y * width + x * 8 + 7 - b) * 4; + sp = (data[y * w + x] >> b & 1) * 3; + dest[dp] = palette[sp]; + dest[dp + 1] = palette[sp + 1]; + dest[dp + 2] = palette[sp + 2]; + dest[dp + 3] = 255; + } + } + + for (let b = 7; b >= 8 - width % 8; b--) { + dp = (y * width + x * 8 + 7 - b) * 4; + sp = (data[y * w + x] >> b & 1) * 3; + dest[dp] = palette[sp]; + dest[dp + 1] = palette[sp + 1]; + dest[dp + 2] = palette[sp + 2]; + dest[dp + 3] = 255; + } + } + + display.blitImage(x, y, width, height, dest, 0, false); + } + + _paletteRect(x, y, width, height, data, palette, display) { + // Convert indexed (palette based) image data to RGB + const dest = this._getScratchBuffer(width * height * 4); + const total = width * height * 4; + for (let i = 0, j = 0; i < total; i += 4, j++) { + const sp = data[j] * 3; + dest[i] = palette[sp]; + dest[i + 1] = palette[sp + 1]; + dest[i + 2] = palette[sp + 2]; + dest[i + 3] = 255; + } + + display.blitImage(x, y, width, height, dest, 0, false); + } + + _gradientFilter(streamId, x, y, width, height, data, display, depth) { + throw new Error("Gradient filter not implemented"); + } + + _readData(data, len_index = 13) { + if (data.length < len_index + 2) { + Log.Error("UDP Decoder, readData, invalid data len") + return null; + } + + + let i = len_index; + let byte = data[i++]; + let len = byte & 0x7f; + // lenth field is variably sized 1 to 3 bytes long + if (byte & 0x80) { + byte = data[i++] + len |= (byte & 0x7f) << 7; + if (byte & 0x80) { + byte = data[i++]; + len |= byte << 14; + } + } + + // get rid of me + if (data.length !== len + i) { + console.log('Rect of size ' + len + ' with data size ' + data.length + ' index of ' + i); + } + + + return data.slice(i, data.length - 1); + } + + _getScratchBuffer(size) { + if (!this._scratchBuffer || (this._scratchBuffer.length < size)) { + this._scratchBuffer = new Uint8Array(size); + } + return this._scratchBuffer; + } +} diff --git a/core/encodings.js b/core/encodings.js index b1bb12b2..9b6d6c23 100644 --- a/core/encodings.js +++ b/core/encodings.js @@ -13,6 +13,7 @@ export const encodings = { encodingHextile: 5, encodingTight: 7, encodingTightPNG: -260, + encodingUDP: -261, pseudoEncodingQualityLevel9: -23, pseudoEncodingQualityLevel0: -32, diff --git a/core/rfb.js b/core/rfb.js index e10fd2a9..b701c663 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -33,6 +33,7 @@ import RREDecoder from "./decoders/rre.js"; import HextileDecoder from "./decoders/hextile.js"; import TightDecoder from "./decoders/tight.js"; import TightPNGDecoder from "./decoders/tightpng.js"; +import UDPDecoder from './decoders/udp.js'; // How many seconds to wait for a disconnect to finish const DISCONNECT_TIMEOUT = 3; @@ -234,6 +235,7 @@ export default class RFB extends EventTargetMixin { this._decoders[encodings.encodingHextile] = new HextileDecoder(); this._decoders[encodings.encodingTight] = new TightDecoder(); this._decoders[encodings.encodingTightPNG] = new TightPNGDecoder(); + this._decoders[encodings.encodingUDP] = new UDPDecoder(); // NB: nothing that needs explicit teardown should be done // before this point, since this can throw an exception @@ -979,54 +981,56 @@ export default class RFB extends EventTargetMixin { if (pieces == 1) { // Handle it immediately Log.Info("Single Piece recieved"); - sock._insertIntoMiddle(u8.slice(12)); - me._handleUdpRect(); + me._handleUdpRect(u8.slice(12)); } else { // Insert into wait array const now = Date.now(); if (udpBuffer.has(id)) { - let item = udpBuffer.get(id); - if (!item) { - Log.Info("Item Missing id: " + id); - return; - } - item.recieved_pieces += 1; - item.data[i] = u8.slice(12); - item.total_bytes += item.data[i].length; - } else { - let item = { - total_pieces: pieces, // number of pieces expected - arrival: now, //time first piece was recieved - recieved_pieces: 1, // current number of pieces in data - total_bytes: 0, // total size of all data pieces combined - data: new Array(pieces) - } - item.data[i] = u8.slice(12); - item.total_bytes = item.data[i].length; - udpBuffer.set(id, item); - } + let item = udpBuffer.get(id); + if (!item) { + Log.Info("Item Missing id: " + id); + return; + } + item.recieved_pieces += 1; + item.data[i] = u8.slice(12); + item.total_bytes += item.data[i].length; + + if (item.total_pieces == item.recieved_pieces) { + // Message is complete, combile data into a single array + var finaldata = new Uint8Array(item.total_bytes); + let z = 0; + for (let x = 0; x < item.data.length; x++) { + finaldata.set(item.data[x], z); + z += item.data[x].length; + } + Log.Info('Completed message applied: ' + finaldata.length + ' ' + item.total_bytes + ' ' + item.total_pieces); + udpBuffer.delete(id); + me._handleUdpRect(finaldata); + } + } else { + let item = { + total_pieces: pieces, // number of pieces expected + arrival: now, //time first piece was recieved + recieved_pieces: 1, // current number of pieces in data + total_bytes: 0, // total size of all data pieces combined + data: new Array(pieces) + } + item.data[i] = u8.slice(12); + item.total_bytes = item.data[i].length; + udpBuffer.set(id, item); + } } + // TODO: this loop is inefficent and likely unneccesary. + // perhaps just keep n number of incomplete messages const now = Date.now(); - for (const [key, value] of udpBuffer.entries()) { - // Drop any messages older than 100ms - if (now - value.arrival > 100) { - Log.Info('Removed id: ' + key); - udpBuffer.delete(key); - } else if (value.total_pieces == value.recieved_pieces) { - // Message is complete, combile data into a single array - var finaldata = new Uint8Array(value.total_bytes); - let z = 0; - for (let x = 0; x < value.data.length; x++) { - finaldata.set(value.data[x], z); - z += value.data[x].length; - } - Log.Info('Completed message applied: ' + finaldata.length + ' ' + value.total_bytes + ' ' + value.total_pieces); - sock._insertIntoMiddle(finaldata); - udpBuffer.delete(key); - me._handleUdpRect(); - } - } + for (const [key, value] of udpBuffer.entries()) { + // Drop any messages older than 100ms + if (now - value.arrival > 100) { + Log.Info('Removed id: ' + key); + udpBuffer.delete(key); + } + } } @@ -2871,32 +2875,39 @@ export default class RFB extends EventTargetMixin { } } - _handleUdpRect() { - if (this._FBU.encoding === null) { - const hdr = this._sock.rQshiftBytes(12); - this._FBU.x = (hdr[0] << 8) + hdr[1]; - this._FBU.y = (hdr[2] << 8) + hdr[3]; - this._FBU.width = (hdr[4] << 8) + hdr[5]; - this._FBU.height = (hdr[6] << 8) + hdr[7]; - this._FBU.encoding = parseInt((hdr[8] << 24) + (hdr[9] << 16) + - (hdr[10] << 8) + hdr[11], 10); - } + _handleUdpRect(data) { + let frame = { + x: (data[0] << 8) + data[1], + y: (data[2] << 8) + data[3], + width: (data[4] << 8) + data[5], + height: (data[6] << 8) + data[7], + encoding: parseInt((data[8] << 24) + (data[9] << 16) + + (data[10] << 8) + data[11], 10) + }; - if (this._FBU.encoding === encodings.pseudoEncodingLastRect) { - } else { - if (!this._handleDataRect()) { + switch (frame.encoding) { + case encodings.pseudoEncodingLastRect: + if (document.visibilityState !== "hidden") { + this._display.flip(); + } + break; + case encodings.encodingTight: + let decoder = this._decoders[encodings.encodingUDP]; + try { + decoder.decodeRect(frame.x, frame.y, + frame.width, frame.height, + data, this._display, + this._fbDepth); + } catch (err) { + this._fail("Error decoding rect: " + err); + return false; + } + break; + default: + Log.Error("Invalid rect encoding via UDP: " + frame.encoding); return false; - } } - if (this._FBU.encoding === encodings.pseudoEncodingLastRect) { - if (document.visibilityState !== "hidden") { - this._display.flip(); - } - } - - this._FBU.encoding = null; - return true; } diff --git a/vnc.html b/vnc.html index 4025bb6b..57815918 100644 --- a/vnc.html +++ b/vnc.html @@ -50,7 +50,7 @@ - +