From 30f9d4eee914b9c7985613319b2c22c6093d8e4b Mon Sep 17 00:00:00 2001 From: lhchavez Date: Sun, 7 Feb 2021 09:47:02 -0800 Subject: [PATCH 1/5] Support grabbing the pointer with the Pointer Lock API This change adds the following: a) A new button on the UI to enter full pointer lock mode, which invokes the Pointer Lock API[1] on the canvas, which hides the cursor and makes mouse events provide relative motion from the previous event (through `movementX` and `movementY`). These can be added to the previously-known mouse position to convert it back to an absolute position. b) Adds support for the VMware Cursor Position pseudo-encoding[2], which servers can use when they make cursor position changes themselves. This is done by some APIs like SDL, when they detect that the client does not support relative mouse movement[3] and then "warp"[4] the cursor to the center of the window, to calculate the relative mouse motion themselves. c) When the canvas is in pointer lock mode and the cursor is not being locally displayed, it updates the cursor position with the information that the server sends, since the actual position of the cursor does not matter locally anymore, since it's not visible. d) Adds some tests for the above. You can try this out end-to-end with TigerVNC with https://github.com/TigerVNC/tigervnc/pull/1198 applied! Fixes: #1493 under some circumstances (at least all SDL games would now work). 1: https://developer.mozilla.org/en-US/docs/Web/API/Pointer_Lock_API 2: https://github.com/rfbproto/rfbproto/blob/master/rfbproto.rst#vmware-cursor-position-pseudo-encoding 3: https://hg.libsdl.org/SDL/file/28e3b60e2131/src/events/SDL_mouse.c#l804 4: https://tronche.com/gui/x/xlib/input/XWarpPointer.html --- app/images/pointer.svg | 78 ++++++++++++++++++++++++++++++++++++++++++ app/ui.js | 45 ++++++++++++++++++++++++ core/encodings.js | 1 + core/rfb.js | 65 ++++++++++++++++++++++++++++++++++- tests/test.rfb.js | 77 +++++++++++++++++++++++++++++++++++++++++ vnc.html | 5 +++ 6 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 app/images/pointer.svg diff --git a/app/images/pointer.svg b/app/images/pointer.svg new file mode 100644 index 00000000..dd394008 --- /dev/null +++ b/app/images/pointer.svg @@ -0,0 +1,78 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + diff --git a/app/ui.js b/app/ui.js index 8e2e78ff..29335572 100644 --- a/app/ui.js +++ b/app/ui.js @@ -224,6 +224,10 @@ const UI = { document.getElementById("noVNC_view_drag_button") .addEventListener('click', UI.toggleViewDrag); + document + .getElementById("noVNC_pointer_lock_button") + .addEventListener("click", UI.requestPointerLock); + document.getElementById("noVNC_control_bar_handle") .addEventListener('mousedown', UI.controlbarHandleMouseDown); document.getElementById("noVNC_control_bar_handle") @@ -441,6 +445,7 @@ const UI = { UI.updatePowerButton(); UI.keepControlbar(); } + UI.updatePointerLockButton(); // State change closes dialogs as they may not be relevant // anymore @@ -1036,6 +1041,7 @@ const UI = { UI.rfb.addEventListener("clipboard", UI.clipboardReceive); UI.rfb.addEventListener("bell", UI.bell); UI.rfb.addEventListener("desktopname", UI.updateDesktopName); + UI.rfb.addEventListener("pointerlock", UI.pointerLockChanged); UI.rfb.clipViewport = UI.getSetting('view_clip'); UI.rfb.scaleViewport = UI.getSetting('resize') === 'scale'; UI.rfb.resizeSession = UI.getSetting('resize') === 'remote'; @@ -1297,6 +1303,33 @@ const UI = { /* ------^------- * /VIEW CLIPPING * ============== + * POINTER LOCK + * ------v------*/ + + updatePointerLockButton() { + // Only show the button if the pointer lock API is properly supported + if ( + UI.connected && + (document.pointerLockElement !== undefined || + document.mozPointerLockElement !== undefined) + ) { + document + .getElementById("noVNC_pointer_lock_button") + .classList.remove("noVNC_hidden"); + } else { + document + .getElementById("noVNC_pointer_lock_button") + .classList.add("noVNC_hidden"); + } + }, + + requestPointerLock() { + UI.rfb.requestPointerLock(); + }, + +/* ------^------- + * /POINTER LOCK + * ============== * VIEWDRAG * ------v------*/ @@ -1662,6 +1695,18 @@ const UI = { document.title = e.detail.name + " - " + PAGE_TITLE; }, + pointerLockChanged(e) { + if (e.detail.pointerlock) { + document + .getElementById("noVNC_pointer_lock_button") + .classList.add("noVNC_selected"); + } else { + document + .getElementById("noVNC_pointer_lock_button") + .classList.remove("noVNC_selected"); + } + }, + bell(e) { if (WebUtil.getConfigVar('bell', 'on') === 'on') { const promise = document.getElementById('noVNC_bell').play(); diff --git a/core/encodings.js b/core/encodings.js index 51c09929..584bd01a 100644 --- a/core/encodings.js +++ b/core/encodings.js @@ -28,6 +28,7 @@ export const encodings = { pseudoEncodingCompressLevel9: -247, pseudoEncodingCompressLevel0: -256, pseudoEncodingVMwareCursor: 0x574d5664, + pseudoEncodingVMwareCursorPosition: 0x574d5666, pseudoEncodingExtendedClipboard: 0xc0a1e5ce }; diff --git a/core/rfb.js b/core/rfb.js index 26cdfcd0..e0f793a0 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -151,6 +151,7 @@ export default class RFB extends EventTargetMixin { this._mousePos = {}; this._mouseButtonMask = 0; this._mouseLastMoveTime = 0; + this._pointerLock = false; this._viewportDragging = false; this._viewportDragPos = {}; this._viewportHasMoved = false; @@ -168,6 +169,7 @@ export default class RFB extends EventTargetMixin { focusCanvas: this._focusCanvas.bind(this), windowResize: this._windowResize.bind(this), handleMouse: this._handleMouse.bind(this), + handlePointerLockChange: this._handlePointerLockChange.bind(this), handleWheel: this._handleWheel.bind(this), handleGesture: this._handleGesture.bind(this), }; @@ -477,6 +479,14 @@ export default class RFB extends EventTargetMixin { this._canvas.blur(); } + requestPointerLock() { + if (this._canvas.requestPointerLock) { + this._canvas.requestPointerLock(); + } else if (this._canvas.mozRequestPointerLock) { + this._canvas.mozRequestPointerLock(); + } + } + clipboardPasteFrom(text) { if (this._rfbConnectionState !== 'connected' || this._viewOnly) { return; } @@ -539,6 +549,8 @@ export default class RFB extends EventTargetMixin { // preventDefault() on mousedown doesn't stop this event for some // reason so we have to explicitly block it this._canvas.addEventListener('contextmenu', this._eventHandlers.handleMouse); + // This needs to be installed in document instead of the canvas. + document.addEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange); // Wheel events this._canvas.addEventListener("wheel", this._eventHandlers.handleWheel); @@ -563,6 +575,7 @@ export default class RFB extends EventTargetMixin { this._canvas.removeEventListener('mousemove', this._eventHandlers.handleMouse); this._canvas.removeEventListener('click', this._eventHandlers.handleMouse); this._canvas.removeEventListener('contextmenu', this._eventHandlers.handleMouse); + document.removeEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange); this._canvas.removeEventListener("mousedown", this._eventHandlers.focusCanvas); this._canvas.removeEventListener("touchstart", this._eventHandlers.focusCanvas); window.removeEventListener('resize', this._eventHandlers.windowResize); @@ -885,8 +898,26 @@ export default class RFB extends EventTargetMixin { return; } - let pos = clientToElement(ev.clientX, ev.clientY, + let pos; + if (this._pointerLock) { + pos = { + x: this._mousePos.x + ev.movementX, + y: this._mousePos.y + ev.movementY, + }; + if (pos.x < 0) { + pos.x = 0; + } else if (pos.x > this._fbWidth) { + pos.x = this._fbWidth; + } + if (pos.y < 0) { + pos.y = 0; + } else if (pos.y > this._fbHeight) { + pos.y = this._fbHeight; + } + } else { + pos = clientToElement(ev.clientX, ev.clientY, this._canvas); + } switch (ev.type) { case 'mousedown': @@ -987,6 +1018,20 @@ export default class RFB extends EventTargetMixin { this._mouseLastMoveTime = Date.now(); } + _handlePointerLockChange() { + if ( + document.pointerLockElement === this._canvas || + document.mozPointerLockElement === this._canvas + ) { + this._pointerLock = true; + } else { + this._pointerLock = false; + } + this.dispatchEvent(new CustomEvent( + "pointerlock", + { detail: { pointerlock: this._pointerLock }, })); + } + _sendMouse(x, y, mask) { if (this._rfbConnectionState !== 'connected') { return; } if (this._viewOnly) { return; } // View only, skip mouse events @@ -1767,6 +1812,8 @@ export default class RFB extends EventTargetMixin { encs.push(encodings.pseudoEncodingCursor); } + encs.push(encodings.pseudoEncodingVMwareCursorPosition); + RFB.messages.clientEncodings(this._sock, encs); } @@ -2165,6 +2212,9 @@ export default class RFB extends EventTargetMixin { case encodings.pseudoEncodingVMwareCursor: return this._handleVMwareCursor(); + case encodings.pseudoEncodingVMwareCursorPosition: + return this._handleVMwareCursorPosition(); + case encodings.pseudoEncodingCursor: return this._handleCursor(); @@ -2303,6 +2353,19 @@ export default class RFB extends EventTargetMixin { return true; } + _handleVMwareCursorPosition() { + const x = this._FBU.x; + const y = this._FBU.y; + + if (this._pointerLock) { + // Only attempt to match the server's pointer position if we are in + // pointer lock mode. + this._mousePos = { x: x, y: y }; + } + + return true; + } + _handleCursor() { const hotx = this._FBU.x; // hotspot-x const hoty = this._FBU.y; // hotspot-y diff --git a/tests/test.rfb.js b/tests/test.rfb.js index d5a9adc8..41c75404 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -2514,6 +2514,27 @@ describe('Remote Frame Buffer Protocol Client', function () { client._canvas.dispatchEvent(ev); } + function supportsSendMouseMovementEvent() { + // Some browsers (like Safari) support the movementX / + // movementY properties of MouseEvent, but do not allow creation + // of non-trusted events with those properties. + let ev; + + ev = new MouseEvent('mousemove', + { 'movementX': 100, + 'movementY': 100 }); + return ev.movementX === 100 && ev.movementY === 100; + } + + function sendMouseMovementEvent(dx, dy) { + let ev; + + ev = new MouseEvent('mousemove', + { 'movementX': dx, + 'movementY': dy }); + client._canvas.dispatchEvent(ev); + } + function sendMouseButtonEvent(x, y, down, button) { let pos = elementToClient(x, y); let ev; @@ -2627,6 +2648,62 @@ describe('Remote Frame Buffer Protocol Client', function () { 50, 70, 0x0); }); + it('should ignore remote cursor position updates', function () { + if (!supportsSendMouseMovementEvent()) { + this.skip(); + return; + } + // Simple VMware Cursor Position FBU message with pointer coordinates + // (50, 50). + const incoming = [ 0x00, 0x00, 0x00, 0x01, 0x00, 0x32, 0x00, 0x32, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x4d, 0x56, 0x66 ]; + client._resize(100, 100); + + const cursorSpy = sinon.spy(client, '_handleVMwareCursorPosition'); + client._sock._websocket._receiveData(new Uint8Array(incoming)); + expect(cursorSpy).to.have.been.calledOnceWith(); + cursorSpy.restore(); + + expect(client._mousePos).to.deep.equal({ }); + sendMouseMoveEvent(10, 10); + clock.tick(100); + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 10, 10, 0x0); + }); + + it('should handle remote mouse position updates in pointer lock mode', function () { + if (!supportsSendMouseMovementEvent()) { + this.skip(); + return; + } + // Simple VMware Cursor Position FBU message with pointer coordinates + // (50, 50). + const incoming = [ 0x00, 0x00, 0x00, 0x01, 0x00, 0x32, 0x00, 0x32, + 0x00, 0x00, 0x00, 0x00, 0x57, 0x4d, 0x56, 0x66 ]; + client._resize(100, 100); + + const spy = sinon.spy(); + client.addEventListener("pointerlock", spy); + let stub = sinon.stub(document, 'pointerLockElement'); + stub.get(function () { return client._canvas; }); + client._handlePointerLockChange(); + stub.restore(); + client._sock._websocket._receiveData(new Uint8Array([0x02, 0x02])); + expect(spy).to.have.been.calledOnce; + expect(spy.args[0][0].detail.pointerlock).to.be.true; + + const cursorSpy = sinon.spy(client, '_handleVMwareCursorPosition'); + client._sock._websocket._receiveData(new Uint8Array(incoming)); + expect(cursorSpy).to.have.been.calledOnceWith(); + cursorSpy.restore(); + + expect(client._mousePos).to.deep.equal({ x: 50, y: 50 }); + sendMouseMovementEvent(10, 10); + clock.tick(100); + expect(pointerEvent).to.have.been.calledOnceWith(client._sock, + 60, 60, 0x0); + }); + describe('Event Aggregation', function () { it('should send a single pointer event on mouse movement', function () { sendMouseMoveEvent(50, 70); diff --git a/vnc.html b/vnc.html index 7870b7c3..5e7fcef2 100644 --- a/vnc.html +++ b/vnc.html @@ -79,6 +79,11 @@ id="noVNC_view_drag_button" class="noVNC_button noVNC_hidden" title="Move/Drag Viewport"> + + +
Date: Mon, 8 Feb 2021 05:13:07 -0800 Subject: [PATCH 2/5] Add documentation This makes it clear that people can call these methods. --- docs/API.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/API.md b/docs/API.md index d78360a2..780001bc 100644 --- a/docs/API.md +++ b/docs/API.md @@ -113,6 +113,10 @@ protocol stream. - The `capabilities` event is fired when `RFB.capabilities` is updated. +[`pointerlock`](#pointerlock) + - The `pointerlock` event is fired when the Pointer Lock is acquired (or + released) by the canvas. + ### Methods [`RFB.disconnect()`](#rfbdisconnect) @@ -146,6 +150,9 @@ protocol stream. [`RFB.clipboardPasteFrom()`](#rfbclipboardPasteFrom) - Send clipboard contents to server. +[`RFB.requestPointerLock()`](#rfbrequestPointerLock) + - Requests that the RFB canvas acquire a Pointer Lock. + ### Details #### RFB() @@ -262,6 +269,15 @@ The `capabilities` event is fired whenever an entry is added or removed from `RFB.capabilities`. The `detail` property is an `Object` with the property `capabilities` containing the new value of `RFB.capabilities`. +#### pointerlock + +The `pointerlock` event is fired when the state of the canvas' Pointer Lock has +changed, either because it has successfully acquired the lock and will have +full control of the mouse pointer, or because the lock was released by the user +pressing the ESC key or performing a browser-specific gesture. The `detail` +property is an `Object` with the property `pointerlock` containing whether the +lock is currently held or not. + #### RFB.disconnect() The `RFB.disconnect()` method is used to disconnect from the currently @@ -383,3 +399,19 @@ to the remote server. **`text`** - A `DOMString` specifying the clipboard data to send. + +#### RFB.requestPointerLock() + +The `RFB.requestPointerLock()` method is used to request that the RFB canvas +hold a [Pointer +Lock](https://developer.mozilla.org/en-US/docs/Web/API/Pointer_Lock_API), which +hides the mouse cursor and provides relative motion events. This must be called +directly from an event handler where a user has directly interacted with the +browser for the browser to allow this. + +If the acquisition of the pointer lock is successful, a `pointerlock` event +will be fired. + +##### Syntax + + RFB.requestPointerLock( ); From 62a0643ed33ba33bab824cef8c0f868e9bb2a110 Mon Sep 17 00:00:00 2001 From: lhchavez Date: Wed, 3 Mar 2021 20:16:34 -0800 Subject: [PATCH 3/5] Emulate the cursor when the pointerlock is engaged This avoids the user having to guess where their pointer is, since the browsers will hide the cursor with no option to unhide it. --- core/rfb.js | 3 +++ core/util/cursor.js | 53 +++++++++++++++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/core/rfb.js b/core/rfb.js index e0f793a0..7fc246ee 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -914,6 +914,7 @@ export default class RFB extends EventTargetMixin { } else if (pos.y > this._fbHeight) { pos.y = this._fbHeight; } + this._cursor.move(pos.x, pos.y); } else { pos = clientToElement(ev.clientX, ev.clientY, this._canvas); @@ -1024,8 +1025,10 @@ export default class RFB extends EventTargetMixin { document.mozPointerLockElement === this._canvas ) { this._pointerLock = true; + this._cursor.setEmulateCursor(true); } else { this._pointerLock = false; + this._cursor.setEmulateCursor(false); } this.dispatchEvent(new CustomEvent( "pointerlock", diff --git a/core/util/cursor.js b/core/util/cursor.js index 12bcceda..0af8617d 100644 --- a/core/util/cursor.js +++ b/core/util/cursor.js @@ -6,21 +6,20 @@ import { supportsCursorURIs, isTouchDevice } from './browser.js'; -const useFallback = !supportsCursorURIs || isTouchDevice; +const needsFallback = !supportsCursorURIs || isTouchDevice; export default class Cursor { constructor() { this._target = null; this._canvas = document.createElement('canvas'); + this._canvas.style.position = 'fixed'; + this._canvas.style.zIndex = '65535'; + this._canvas.style.pointerEvents = 'none'; + // Can't use "display" because of Firefox bug #1445997 + this._canvas.style.visibility = 'hidden'; - if (useFallback) { - this._canvas.style.position = 'fixed'; - this._canvas.style.zIndex = '65535'; - this._canvas.style.pointerEvents = 'none'; - // Can't use "display" because of Firefox bug #1445997 - this._canvas.style.visibility = 'hidden'; - } + this._useFallback = needsFallback; this._position = { x: 0, y: 0 }; this._hotSpot = { x: 0, y: 0 }; @@ -40,9 +39,14 @@ export default class Cursor { this._target = target; - if (useFallback) { - document.body.appendChild(this._canvas); + document.body.appendChild(this._canvas); + if (needsFallback) { + // Only add the event listeners if this will be responsible for + // rendering the cursor all the time. Otherwise, the cursor will + // only be rendered then the forced emulation is turned on, and + // that doesn't require this class to be adjusting the cursor + // position. const options = { capture: true, passive: true }; this._target.addEventListener('mouseover', this._eventHandlers.mouseover, options); this._target.addEventListener('mouseleave', this._eventHandlers.mouseleave, options); @@ -58,16 +62,16 @@ export default class Cursor { return; } - if (useFallback) { + if (needsFallback) { const options = { capture: true, passive: true }; this._target.removeEventListener('mouseover', this._eventHandlers.mouseover, options); this._target.removeEventListener('mouseleave', this._eventHandlers.mouseleave, options); this._target.removeEventListener('mousemove', this._eventHandlers.mousemove, options); this._target.removeEventListener('mouseup', this._eventHandlers.mouseup, options); - - document.body.removeChild(this._canvas); } + document.body.removeChild(this._canvas); + this._target = null; } @@ -91,9 +95,10 @@ export default class Cursor { ctx.clearRect(0, 0, w, h); ctx.putImageData(img, 0, 0); - if (useFallback) { + if (this._useFallback || needsFallback) { this._updatePosition(); - } else { + } + if (!needsFallback) { let url = this._canvas.toDataURL(); this._target.style.cursor = 'url(' + url + ')' + hotx + ' ' + hoty + ', default'; } @@ -112,7 +117,7 @@ export default class Cursor { // Mouse events might be emulated, this allows // moving the cursor in such cases move(clientX, clientY) { - if (!useFallback) { + if (!this._useFallback) { return; } // clientX/clientY are relative the _visual viewport_, @@ -130,6 +135,22 @@ export default class Cursor { this._updateVisibility(target); } + // Force the use of cursor emulation. This is needed when the pointer lock + // is in use, since the browser will not render the cursor. + setEmulateCursor(emulate) { + if (needsFallback) { + // We need to use the fallback all the time, so we shouldn't update + // the fallback flag. + return; + } + this._useFallback = emulate; + if (this._useFallback) { + this._showCursor(); + } else { + this._hideCursor(); + } + } + _handleMouseOver(event) { // This event could be because we're entering the target, or // moving around amongst its sub elements. Let the move handler From 30fda3c9481124ccb1da965db953245a2db7048d Mon Sep 17 00:00:00 2001 From: lhchavez Date: Tue, 16 Mar 2021 06:59:56 -0700 Subject: [PATCH 4/5] Use "input lock" rather than pointer lock This new API can now be used to support [keyboard lock](https://web.dev/keyboard-lock/), although support for that is limited to Chrome only at the moment. --- app/ui.js | 8 ++++---- core/rfb.js | 49 +++++++++++++++++++++++++++++++++++---------- docs/API.md | 51 ++++++++++++++++++++++++++--------------------- tests/test.rfb.js | 4 ++-- 4 files changed, 73 insertions(+), 39 deletions(-) diff --git a/app/ui.js b/app/ui.js index 29335572..ee831961 100644 --- a/app/ui.js +++ b/app/ui.js @@ -1041,7 +1041,7 @@ const UI = { UI.rfb.addEventListener("clipboard", UI.clipboardReceive); UI.rfb.addEventListener("bell", UI.bell); UI.rfb.addEventListener("desktopname", UI.updateDesktopName); - UI.rfb.addEventListener("pointerlock", UI.pointerLockChanged); + UI.rfb.addEventListener("inputlock", UI.inputLockChanged); UI.rfb.clipViewport = UI.getSetting('view_clip'); UI.rfb.scaleViewport = UI.getSetting('resize') === 'scale'; UI.rfb.resizeSession = UI.getSetting('resize') === 'remote'; @@ -1324,7 +1324,7 @@ const UI = { }, requestPointerLock() { - UI.rfb.requestPointerLock(); + UI.rfb.requestInputLock({ pointer: true }); }, /* ------^------- @@ -1695,8 +1695,8 @@ const UI = { document.title = e.detail.name + " - " + PAGE_TITLE; }, - pointerLockChanged(e) { - if (e.detail.pointerlock) { + inputLockChanged(e) { + if (e.detail.pointer) { document .getElementById("noVNC_pointer_lock_button") .classList.add("noVNC_selected"); diff --git a/core/rfb.js b/core/rfb.js index 7fc246ee..1ef01833 100644 --- a/core/rfb.js +++ b/core/rfb.js @@ -170,6 +170,7 @@ export default class RFB extends EventTargetMixin { windowResize: this._windowResize.bind(this), handleMouse: this._handleMouse.bind(this), handlePointerLockChange: this._handlePointerLockChange.bind(this), + handlePointerLockError: this._handlePointerLockError.bind(this), handleWheel: this._handleWheel.bind(this), handleGesture: this._handleGesture.bind(this), }; @@ -479,12 +480,22 @@ export default class RFB extends EventTargetMixin { this._canvas.blur(); } - requestPointerLock() { - if (this._canvas.requestPointerLock) { - this._canvas.requestPointerLock(); - } else if (this._canvas.mozRequestPointerLock) { - this._canvas.mozRequestPointerLock(); + requestInputLock(locks) { + if (locks.pointer) { + if (this._canvas.requestPointerLock) { + this._canvas.requestPointerLock(); + return; + } + if (this._canvas.mozRequestPointerLock) { + this._canvas.mozRequestPointerLock(); + return; + } } + // If we were not able to request any lock, still let the user know + // about the result. + this.dispatchEvent(new CustomEvent( + "inputlock", + { detail: { pointer: this._pointerLock }, })); } clipboardPasteFrom(text) { @@ -549,8 +560,14 @@ export default class RFB extends EventTargetMixin { // preventDefault() on mousedown doesn't stop this event for some // reason so we have to explicitly block it this._canvas.addEventListener('contextmenu', this._eventHandlers.handleMouse); - // This needs to be installed in document instead of the canvas. - document.addEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange); + // Pointer Lock listeners need to be installed in document instead of the canvas. + if (document.onpointerlockchange !== undefined) { + document.addEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange, false); + document.addEventListener('pointerlockerror', this._eventHandlers.handlePointerLockError, false); + } else if (document.onmozpointerlockchange !== undefined) { + document.addEventListener('mozpointerlockchange', this._eventHandlers.handlePointerLockChange, false); + document.addEventListener('mozpointerlockerror', this._eventHandlers.handlePointerLockError, false); + } // Wheel events this._canvas.addEventListener("wheel", this._eventHandlers.handleWheel); @@ -575,7 +592,13 @@ export default class RFB extends EventTargetMixin { this._canvas.removeEventListener('mousemove', this._eventHandlers.handleMouse); this._canvas.removeEventListener('click', this._eventHandlers.handleMouse); this._canvas.removeEventListener('contextmenu', this._eventHandlers.handleMouse); - document.removeEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange); + if (document.onpointerlockchange !== undefined) { + document.removeEventListener('pointerlockchange', this._eventHandlers.handlePointerLockChange); + document.removeEventListener('pointerlockerror', this._eventHandlers.handlePointerLockError); + } else if (document.onmozpointerlockchange !== undefined) { + document.removeEventListener('mozpointerlockchange', this._eventHandlers.handlePointerLockChange); + document.removeEventListener('mozpointerlockerror', this._eventHandlers.handlePointerLockError); + } this._canvas.removeEventListener("mousedown", this._eventHandlers.focusCanvas); this._canvas.removeEventListener("touchstart", this._eventHandlers.focusCanvas); window.removeEventListener('resize', this._eventHandlers.windowResize); @@ -1031,8 +1054,14 @@ export default class RFB extends EventTargetMixin { this._cursor.setEmulateCursor(false); } this.dispatchEvent(new CustomEvent( - "pointerlock", - { detail: { pointerlock: this._pointerLock }, })); + "inputlock", + { detail: { pointer: this._pointerLock }, })); + } + + _handlePointerLockError() { + this.dispatchEvent(new CustomEvent( + "inputlock", + { detail: { pointer: this._pointerLock }, })); } _sendMouse(x, y, mask) { diff --git a/docs/API.md b/docs/API.md index 780001bc..134a9709 100644 --- a/docs/API.md +++ b/docs/API.md @@ -113,9 +113,9 @@ protocol stream. - The `capabilities` event is fired when `RFB.capabilities` is updated. -[`pointerlock`](#pointerlock) - - The `pointerlock` event is fired when the Pointer Lock is acquired (or - released) by the canvas. +[`inputlock`](#inputlock) + - The `inputlock` event is fired when an input lock is acquired (or released) + by the canvas. ### Methods @@ -150,8 +150,8 @@ protocol stream. [`RFB.clipboardPasteFrom()`](#rfbclipboardPasteFrom) - Send clipboard contents to server. -[`RFB.requestPointerLock()`](#rfbrequestPointerLock) - - Requests that the RFB canvas acquire a Pointer Lock. +[`RFB.requestInputLock()`](#rfbrequestInputLock) + - Requests that the RFB canvas acquire an input lock. ### Details @@ -269,14 +269,14 @@ The `capabilities` event is fired whenever an entry is added or removed from `RFB.capabilities`. The `detail` property is an `Object` with the property `capabilities` containing the new value of `RFB.capabilities`. -#### pointerlock +#### inputlock -The `pointerlock` event is fired when the state of the canvas' Pointer Lock has -changed, either because it has successfully acquired the lock and will have -full control of the mouse pointer, or because the lock was released by the user -pressing the ESC key or performing a browser-specific gesture. The `detail` -property is an `Object` with the property `pointerlock` containing whether the -lock is currently held or not. +The `inputlock` event is fired after a request to acquire an input lock or +whenever the state of the canvas' input lock has changed, the latter typically +occurs because the lock was released by the user pressing the ESC key or +performing a browser-specific gesture. The `detail` property is an `Object` +with the property `pointer` containing whether the Pointer Lock is currently +held or not. #### RFB.disconnect() @@ -400,18 +400,23 @@ to the remote server. **`text`** - A `DOMString` specifying the clipboard data to send. -#### RFB.requestPointerLock() +#### RFB.requestInputLock() -The `RFB.requestPointerLock()` method is used to request that the RFB canvas -hold a [Pointer -Lock](https://developer.mozilla.org/en-US/docs/Web/API/Pointer_Lock_API), which -hides the mouse cursor and provides relative motion events. This must be called -directly from an event handler where a user has directly interacted with the -browser for the browser to allow this. - -If the acquisition of the pointer lock is successful, a `pointerlock` event -will be fired. +The `RFB.requestInputLock()` method is used to request that the RFB canvas hold +an input lock. An `inputlock` event will be fired with the result of the +acquisition of the requested locks. ##### Syntax - RFB.requestPointerLock( ); + RFB.requestInputLock( { pointer: true } ); + +###### Parameters + +**`pointer`** + - Requests to acquire a [Pointer + Lock](https://developer.mozilla.org/en-US/docs/Web/API/Pointer_Lock_API), + which hides the local mouse cursor and provides relative motion events. + This must be called directly from an event handler where a user has + directly interacted with an element through an [engagement + gesture](https://w3c.github.io/pointerlock/#dfn-engagement-gesture) (e.g. a + click or touch event) for the browser to allow this. diff --git a/tests/test.rfb.js b/tests/test.rfb.js index 41c75404..c71c273a 100644 --- a/tests/test.rfb.js +++ b/tests/test.rfb.js @@ -2683,14 +2683,14 @@ describe('Remote Frame Buffer Protocol Client', function () { client._resize(100, 100); const spy = sinon.spy(); - client.addEventListener("pointerlock", spy); + client.addEventListener("inputlock", spy); let stub = sinon.stub(document, 'pointerLockElement'); stub.get(function () { return client._canvas; }); client._handlePointerLockChange(); stub.restore(); client._sock._websocket._receiveData(new Uint8Array([0x02, 0x02])); expect(spy).to.have.been.calledOnce; - expect(spy.args[0][0].detail.pointerlock).to.be.true; + expect(spy.args[0][0].detail.pointer).to.be.true; const cursorSpy = sinon.spy(client, '_handleVMwareCursorPosition'); client._sock._websocket._receiveData(new Uint8Array(incoming)); From a7f4d708cab667e47fc189cc3da213e441bc0ba6 Mon Sep 17 00:00:00 2001 From: lhchavez Date: Tue, 16 Mar 2021 07:07:14 -0700 Subject: [PATCH 5/5] Hide the pointer capture button unless in fullscreen This change is a compromise to de-clutter the navbar by only showing the pointer capture button when fullscreen is enabled. There is no strong requirement (from the browser side) to be in fullscreen to acquire a pointer lock. --- app/ui.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/ui.js b/app/ui.js index ee831961..714c0dec 100644 --- a/app/ui.js +++ b/app/ui.js @@ -1251,6 +1251,7 @@ const UI = { document.getElementById('noVNC_fullscreen_button') .classList.remove("noVNC_selected"); } + UI.updatePointerLockButton(); }, /* ------^------- @@ -1308,8 +1309,13 @@ const UI = { updatePointerLockButton() { // Only show the button if the pointer lock API is properly supported + // AND in fullscreen. if ( UI.connected && + (document.fullscreenElement || // alternative standard method + document.mozFullScreenElement || // currently working methods + document.webkitFullscreenElement || + document.msFullscreenElement) && (document.pointerLockElement !== undefined || document.mozPointerLockElement !== undefined) ) {