diff --git a/app/images/record.svg b/app/images/record.svg new file mode 100644 index 00000000..27ea946f --- /dev/null +++ b/app/images/record.svg @@ -0,0 +1,13 @@ + + + + diff --git a/app/styles/base.css b/app/styles/base.css index 33f0f359..e913f3a6 100644 --- a/app/styles/base.css +++ b/app/styles/base.css @@ -605,6 +605,22 @@ html { max-height: calc(100vh - 10em - 25px); } +/* Recording */ +#noVNC_record_button.noVNC_recording { + border-color: #ff4444; + background-color: rgba(255, 68, 68, 0.3); +} +#noVNC_record_button.noVNC_recording img { + filter: brightness(0) saturate(100%) invert(35%) sepia(100%) saturate(3000%) hue-rotate(340deg) brightness(100%) contrast(100%); +} +#noVNC_record p { + margin: 5px 0; +} +#noVNC_record input[type=button] { + width: 100%; + margin-top: 5px; +} + /* Settings */ #noVNC_settings { } diff --git a/app/ui.js b/app/ui.js index 51e57bd3..25ec000a 100644 --- a/app/ui.js +++ b/app/ui.js @@ -34,6 +34,16 @@ const UI = { idleControlbarTimeout: null, closeControlbarTimeout: null, + // Recording state + recording: false, + recordingPending: false, // True if recording should start on next connect + recordingStartTime: null, + recordingFrameCount: 0, + recordingBytesWritten: 0, + recordingFileHandle: null, + recordingWritable: null, + recordingWriteQueue: Promise.resolve(), // Chain writes to ensure order + controlbarGrabbed: false, controlbarDrag: false, controlbarMouseDownClientY: 0, @@ -121,6 +131,7 @@ const UI = { UI.addConnectionControlHandlers(); UI.addClipboardHandlers(); UI.addSettingsHandlers(); + UI.addRecordingHandlers(); document.getElementById("noVNC_status") .addEventListener('click', UI.hideStatus); @@ -133,6 +144,14 @@ const UI = { document.documentElement.classList.remove("noVNC_loading"); + // Check for autorecord setting + let autorecord = UI.getSetting('autorecord'); + if (autorecord === 'true' || autorecord == '1') { + UI.recordingPending = true; + document.getElementById('noVNC_record_button').classList.add('noVNC_selected'); + document.getElementById('noVNC_record_button').classList.add('noVNC_recording'); + } + let autoconnect = UI.getSetting('autoconnect'); if (autoconnect === 'true' || autoconnect == '1') { autoconnect = true; @@ -189,6 +208,7 @@ const UI = { UI.initSetting('repeaterID', ''); UI.initSetting('reconnect', false); UI.initSetting('reconnect_delay', 5000); + UI.initSetting('autorecord', false); }, // Adds a link to the label elements on the corresponding input elements setupSettingLabels() { @@ -381,6 +401,32 @@ const UI = { UI.addSettingChangeHandler('reconnect_delay'); }, + addRecordingHandlers() { + document.getElementById("noVNC_record_button") + .addEventListener('click', UI.toggleRecordingPanel); + document.getElementById("noVNC_record_start_button") + .addEventListener('click', UI.startRecording); + document.getElementById("noVNC_record_stop_button") + .addEventListener('click', UI.stopRecording); + document.getElementById("noVNC_record_download_button") + .addEventListener('click', UI.downloadRecording); + + // Ensure recording is properly closed when page unloads + const closeRecordingFile = () => { + if (UI.recording && UI.recordingWritable) { + UI.recording = false; + UI.recordingWriteQueue.then(() => { + if (UI.recordingWritable) { + UI.recordingWritable.close(); + UI.recordingWritable = null; + } + }); + } + }; + window.addEventListener('beforeunload', closeRecordingFile); + window.addEventListener('pagehide', closeRecordingFile); + }, + addFullscreenHandlers() { document.getElementById("noVNC_fullscreen_button") .addEventListener('click', UI.toggleFullscreen); @@ -867,6 +913,7 @@ const UI = { UI.closePowerPanel(); UI.closeClipboardPanel(); UI.closeExtraKeys(); + UI.closeRecordingPanel(); }, /* ------^------- @@ -1010,6 +1057,282 @@ const UI = { /* ------^------- * /CLIPBOARD * ============== + * RECORDING + * ------v------*/ + + openRecordingPanel() { + UI.closeAllPanels(); + UI.openControlbar(); + + UI.updateRecordingStats(); + + document.getElementById('noVNC_record') + .classList.add("noVNC_open"); + document.getElementById('noVNC_record_button') + .classList.add("noVNC_selected"); + }, + + closeRecordingPanel() { + document.getElementById('noVNC_record') + .classList.remove("noVNC_open"); + document.getElementById('noVNC_record_button') + .classList.remove("noVNC_selected"); + }, + + toggleRecordingPanel() { + if (document.getElementById('noVNC_record') + .classList.contains("noVNC_open")) { + UI.closeRecordingPanel(); + } else { + UI.openRecordingPanel(); + } + }, + + updateRecordingStats() { + const statusElem = document.getElementById('noVNC_record_status'); + const statsElem = document.getElementById('noVNC_record_stats'); + + if (UI.recording) { + const elapsed = Math.floor((Date.now() - UI.recordingStartTime) / 1000); + const minutes = Math.floor(elapsed / 60); + const seconds = elapsed % 60; + statusElem.textContent = `Recording: ${minutes}:${seconds.toString().padStart(2, '0')}`; + statusElem.style.color = '#ff4444'; + + const mbytes = (UI.recordingBytesWritten / (1024 * 1024)).toFixed(2); + statsElem.textContent = `Frames: ${UI.recordingFrameCount}, Size: ${mbytes} MB (OPFS)`; + } else if (UI.recordingPending) { + statusElem.textContent = 'Waiting for connection...'; + statusElem.style.color = '#ffaa00'; + statsElem.textContent = 'Recording will start when you connect'; + } else { + statusElem.textContent = 'Not recording'; + statusElem.style.color = ''; + + if (UI.recordingFrameCount > 0) { + const mbytes = (UI.recordingBytesWritten / (1024 * 1024)).toFixed(2); + statsElem.textContent = `Recorded: ${UI.recordingFrameCount} frames, ${mbytes} MB`; + } else { + statsElem.textContent = ''; + } + } + + // Update storage usage (async) + UI.updateStorageUsage(); + }, + + async updateStorageUsage() { + const storageElem = document.getElementById('noVNC_record_storage'); + if (!storageElem) return; + + try { + const s = await navigator.storage.estimate(); + const usageGB = (s.usage / 1e9).toFixed(2); + const quotaGB = (s.quota / 1e9).toFixed(1); + const percent = (s.usage / s.quota * 100).toFixed(1); + storageElem.textContent = `Storage: ${usageGB}GB / ${quotaGB}GB (${percent}%)`; + } catch (e) { + storageElem.textContent = 'Storage: unavailable'; + } + }, + + async startRecording() { + if (UI.recording) { + return; + } + + // If already connected, user needs to reconnect to capture from beginning + if (UI.connected) { + UI.showStatus(_("Disconnect and reconnect to record from the beginning"), 'warn'); + return; + } + + // Delete any existing recording file + try { + const root = await navigator.storage.getDirectory(); + await root.removeEntry('vnc-recording.bin'); + } catch (e) { + // File doesn't exist, that's fine + } + + // Set pending - recording will start when connection is made + UI.recordingPending = true; + UI.recordingFrameCount = 0; + UI.recordingBytesWritten = 0; + + // Update UI to show pending state + document.getElementById('noVNC_record_button').classList.add('noVNC_selected'); + document.getElementById('noVNC_record_button').classList.add('noVNC_recording'); + document.getElementById('noVNC_record_start_button').disabled = true; + document.getElementById('noVNC_record_stop_button').disabled = false; + document.getElementById('noVNC_record_download_button').disabled = true; + document.getElementById('noVNC_record_status').textContent = 'Waiting for connection...'; + document.getElementById('noVNC_record_status').style.color = '#ffaa00'; + + UI.showStatus(_("Recording will start when you connect"), 'normal'); + }, + + async stopRecording() { + if (!UI.recording && !UI.recordingPending) { + return; + } + + Log.Info("Stopping recording, captured " + UI.recordingFrameCount + " frames"); + + // Setting recording to false stops the wrapped handlers from capturing + UI.recording = false; + UI.recordingPending = false; + + // Wait for pending writes and close the OPFS file + if (UI.recordingWritable) { + try { + await UI.recordingWriteQueue; // Wait for all pending writes + await UI.recordingWritable.close(); + } catch (e) { + Log.Error("Error closing recording file: " + e); + } + UI.recordingWritable = null; + UI.recordingFileHandle = null; + } + + // Stop stats update + if (UI.recordingStatsInterval) { + clearInterval(UI.recordingStatsInterval); + UI.recordingStatsInterval = null; + } + + // Update UI + document.getElementById('noVNC_record_button').classList.remove('noVNC_recording'); + if (!document.getElementById('noVNC_record').classList.contains('noVNC_open')) { + document.getElementById('noVNC_record_button').classList.remove('noVNC_selected'); + } + document.getElementById('noVNC_record_start_button').disabled = false; + document.getElementById('noVNC_record_stop_button').disabled = true; + document.getElementById('noVNC_record_download_button').disabled = (UI.recordingFrameCount === 0); + + UI.updateRecordingStats(); + + UI.showStatus(_("Recording stopped: ") + UI.recordingFrameCount + _(" frames captured"), 'normal'); + }, + + async downloadRecording() { + if (UI.recordingFrameCount === 0) { + UI.showStatus(_("No recording to download"), 'error'); + return; + } + + Log.Info("Generating recording file with " + UI.recordingFrameCount + " frames"); + UI.showStatus(_("Preparing download..."), 'normal'); + + try { + // Open the OPFS recording file for reading + const root = await navigator.storage.getDirectory(); + const fileHandle = await root.getFileHandle('vnc-recording.bin'); + const file = await fileHandle.getFile(); + + // Create a streaming response that converts binary to JS format + const reader = file.stream().getReader(); + const textEncoder = new TextEncoder(); + + let buffer = new Uint8Array(0); + let framesProcessed = 0; + let headerWritten = false; + + const jsStream = new ReadableStream({ + async pull(controller) { + // Write header first + if (!headerWritten) { + const header = '/* noVNC recording - generated by noVNC client-side recorder */\n' + + '/* eslint-disable */\n' + + 'var VNC_frame_data = [\n'; + controller.enqueue(textEncoder.encode(header)); + headerWritten = true; + } + + // Read and process frames + while (true) { + // Try to parse a frame from buffer + // Binary format: fromClient(1) + timestamp(4) + dataLen(4) + data(dataLen) + if (buffer.length >= 9) { + const fromClient = buffer[0] === 1; + const timestamp = (buffer[1] << 24) | (buffer[2] << 16) | (buffer[3] << 8) | buffer[4]; + const dataLen = (buffer[5] << 24) | (buffer[6] << 16) | (buffer[7] << 8) | buffer[8]; + + if (buffer.length >= 9 + dataLen) { + // We have a complete frame + const data = buffer.slice(9, 9 + dataLen); + buffer = buffer.slice(9 + dataLen); + + // Convert to JS format + const prefix = fromClient ? '}' : '{'; + let binary = ''; + for (let j = 0; j < data.length; j++) { + binary += String.fromCharCode(data[j]); + } + const base64Data = btoa(binary); + const frameStr = prefix + timestamp + '{' + base64Data; + const escaped = JSON.stringify(frameStr); + const line = escaped + ',\n'; + + controller.enqueue(textEncoder.encode(line)); + framesProcessed++; + + // Update status periodically + if (framesProcessed % 1000 === 0) { + UI.showStatus(_("Processing: ") + framesProcessed + "/" + UI.recordingFrameCount + _(" frames"), 'normal'); + } + + continue; // Try to parse another frame + } + } + + // Need more data + const { done, value } = await reader.read(); + if (done) { + // Write footer and close + controller.enqueue(textEncoder.encode('"EOF"\n];\n')); + controller.close(); + return; + } + + // Append to buffer + const newBuffer = new Uint8Array(buffer.length + value.length); + newBuffer.set(buffer); + newBuffer.set(value, buffer.length); + buffer = newBuffer; + } + } + }); + + // Create blob from stream and trigger download + const response = new Response(jsStream); + const blob = await response.blob(); + + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + + const now = new Date(); + const dateStr = now.toISOString().replace(/[:.]/g, '-').slice(0, 19); + a.download = 'vnc-recording-' + dateStr + '.js'; + + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + + setTimeout(() => URL.revokeObjectURL(url), 10000); + + UI.showStatus(_("Recording downloaded: ") + framesProcessed + _(" frames"), 'normal'); + + } catch (e) { + Log.Error("Error downloading recording: " + e); + UI.showStatus(_("Download error: ") + e.message, 'error'); + } + }, + +/* ------^------- + * /RECORDING + * ============== * CONNECTION * ------v------*/ @@ -1023,7 +1346,7 @@ const UI = { .classList.remove("noVNC_open"); }, - connect(event, password) { + async connect(event, password) { // Ignore when rfb already exists if (typeof UI.rfb !== 'undefined') { @@ -1072,6 +1395,98 @@ const UI = { url.protocol = (window.location.protocol === "https:") ? 'wss:' : 'ws:'; } + // If recording is pending, set up OPFS file and wrap WebSocket + let OriginalWebSocket = null; + if (UI.recordingPending) { + try { + // Set up OPFS file for recording + const root = await navigator.storage.getDirectory(); + UI.recordingFileHandle = await root.getFileHandle('vnc-recording.bin', { create: true }); + UI.recordingWritable = await UI.recordingFileHandle.createWritable(); + UI.recordingWriteQueue = Promise.resolve(); + + UI.recording = true; + UI.recordingPending = false; + UI.recordingStartTime = Date.now(); + UI.recordingFrameCount = 0; + UI.recordingBytesWritten = 0; + + // Helper to write a frame to OPFS (binary format) + // Format: fromClient(1) + timestamp(4) + dataLen(4) + data(dataLen) + const writeFrame = (fromClient, timestamp, data) => { + if (!UI.recording || !UI.recordingWritable) return; + + const header = new Uint8Array(9); + header[0] = fromClient ? 1 : 0; + header[1] = (timestamp >> 24) & 0xff; + header[2] = (timestamp >> 16) & 0xff; + header[3] = (timestamp >> 8) & 0xff; + header[4] = timestamp & 0xff; + header[5] = (data.length >> 24) & 0xff; + header[6] = (data.length >> 16) & 0xff; + header[7] = (data.length >> 8) & 0xff; + header[8] = data.length & 0xff; + + // Chain writes to ensure order + UI.recordingWriteQueue = UI.recordingWriteQueue.then(async () => { + try { + await UI.recordingWritable.write(header); + await UI.recordingWritable.write(data); + UI.recordingFrameCount++; + UI.recordingBytesWritten += 9 + data.length; + } catch (e) { + Log.Error("Error writing frame: " + e); + } + }); + }; + + // Wrap WebSocket constructor temporarily + OriginalWebSocket = window.WebSocket; + window.WebSocket = function(wsUrl, protocols) { + const ws = new OriginalWebSocket(wsUrl, protocols); + + // Capture server-to-client messages + ws.addEventListener('message', function(e) { + if (UI.recording) { + const timestamp = Date.now() - UI.recordingStartTime; + const data = new Uint8Array(e.data); + writeFrame(false, timestamp, data); + } + }); + + // Wrap send for client-to-server messages + const originalSend = ws.send.bind(ws); + ws.send = function(data) { + if (UI.recording) { + const timestamp = Date.now() - UI.recordingStartTime; + let u8data; + if (data instanceof ArrayBuffer) { + u8data = new Uint8Array(data); + } else if (data instanceof Uint8Array) { + u8data = data; + } else { + u8data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + } + writeFrame(true, timestamp, u8data); + } + originalSend(data); + }; + + return ws; + }; + // Copy static properties + window.WebSocket.CONNECTING = OriginalWebSocket.CONNECTING; + window.WebSocket.OPEN = OriginalWebSocket.OPEN; + window.WebSocket.CLOSING = OriginalWebSocket.CLOSING; + window.WebSocket.CLOSED = OriginalWebSocket.CLOSED; + + } catch (e) { + Log.Error("Failed to set up recording: " + e); + UI.showStatus(_("Recording setup failed: ") + e.message, 'error'); + UI.recordingPending = false; + } + } + try { UI.rfb = new RFB(document.getElementById('noVNC_container'), url.href, @@ -1085,6 +1500,28 @@ const UI = { return; } + // Restore original WebSocket if we wrapped it + if (OriginalWebSocket) { + window.WebSocket = OriginalWebSocket; + // Update recording UI + document.getElementById('noVNC_record_button').classList.add('noVNC_selected'); + document.getElementById('noVNC_record_button').classList.add('noVNC_recording'); + document.getElementById('noVNC_record_start_button').disabled = true; + document.getElementById('noVNC_record_stop_button').disabled = false; + document.getElementById('noVNC_record_download_button').disabled = true; + + UI.updateRecordingStats(); + + // Start periodic stats update + UI.recordingStatsInterval = setInterval(() => { + if (document.getElementById('noVNC_record').classList.contains('noVNC_open')) { + UI.updateRecordingStats(); + } + }, 1000); + + UI.showStatus(_("Recording started (OPFS)"), 'normal'); + } + UI.rfb.addEventListener("connect", UI.connectFinished); UI.rfb.addEventListener("disconnect", UI.disconnectFinished); UI.rfb.addEventListener("serververification", UI.serverVerify); @@ -1106,6 +1543,11 @@ const UI = { }, disconnect() { + // Stop recording if active or pending + if (UI.recording || UI.recordingPending) { + UI.stopRecording(); + } + UI.rfb.disconnect(); UI.connected = false; @@ -1161,6 +1603,11 @@ const UI = { disconnectFinished(e) { const wasConnected = UI.connected; + // Stop recording if active or pending (handles unexpected disconnects) + if (UI.recording || UI.recordingPending) { + UI.stopRecording(); + } + // This variable is ideally set when disconnection starts, but // when the disconnection isn't clean or if it is initiated by // the server, we need to do it here as well since diff --git a/demo_collector.html b/demo_collector.html new file mode 100644 index 00000000..67b6818b --- /dev/null +++ b/demo_collector.html @@ -0,0 +1,413 @@ + + + + + + Demonstration Collector + + + +

Demonstration Collector

+

Record VNC sessions for training data

+ +
+
+ + +
+
+ + +
+ + + + + +
+
Status
+
Ready
+
+
+
+ + + + diff --git a/tests/benchmark_buffer.html b/tests/benchmark_buffer.html new file mode 100644 index 00000000..45426ea8 --- /dev/null +++ b/tests/benchmark_buffer.html @@ -0,0 +1,736 @@ + + + + Recording Buffer Benchmark + + + +

Recording Buffer Benchmark

+

Test different storage methods for high-throughput VNC recording (~10MB/s)

+ +
+ + + +
+ +
+ +
+

1. In-Memory Array

+

Current approach: Push Uint8Array chunks to a JavaScript array. Simple but memory-heavy.

+ + + + +
+
Ready
+
+ + +
+

2. Pre-allocated Buffer

+

Allocate a large ArrayBuffer upfront and write into it. Avoids repeated allocations.

+ + + + +
+
Ready
+
+ + +
+

3. Chunked Arrays (64MB blocks)

+

Store data in fixed-size 64MB ArrayBuffers. Balances memory efficiency with GC pressure.

+ + + + +
+
Ready
+
+ + +
+

4. File Stream (OPFS)

+

Stream to Origin Private File System. Keeps memory low but requires async writes.

+ + + + +
+
Ready
+
+ + +
+

5. IndexedDB Chunks

+

Store chunks in IndexedDB. Persistent storage with automatic memory management.

+ + + + +
+
Ready
+
+ + +
+

6. Blob Accumulation

+

Periodically merge chunks into Blobs. Leverages browser's blob storage.

+ + + + +
+
Ready
+
+
+ +
+

Results

+ + + + + + + + + + + + + +
MethodStatusData WrittenTimeThroughputPeak Memory
+
+ + + + diff --git a/tests/playback-ui.js b/tests/playback-ui.js index b1f263ea..aa49b9c7 100644 --- a/tests/playback-ui.js +++ b/tests/playback-ui.js @@ -116,6 +116,7 @@ class IterationPlayer { this.onfinish = () => {}; this.oniterationfinish = () => {}; this.rfbdisconnected = () => {}; + this.onclientevent = () => {}; } start(realtime) { @@ -130,6 +131,7 @@ class IterationPlayer { _nextIteration() { const player = new RecordingPlayer(this._frames, this._disconnected.bind(this)); player.onfinish = this._iterationFinish.bind(this); + player.onclientevent = this.onclientevent; if (this._state !== 'running') { return; } @@ -212,6 +214,28 @@ function start() { document.getElementById('startButton').disabled = false; document.getElementById('startButton').value = "Start"; }; + // Log client input events (skip mouse moves and framebuffer requests to avoid spam) + player.onclientevent = (timestamp, event) => { + const timeStr = (timestamp / 1000).toFixed(2) + 's'; + switch (event.type) { + case 'KeyEvent': + const action = event.down ? 'down' : 'up'; + message(`[${timeStr}] Key ${action}: ${event.keyName}`); + break; + case 'PointerEvent': + // Only log button down/up, not moves + if (!event.isMove && event.events.length > 0) { + for (const e of event.events) { + message(`[${timeStr}] Mouse ${e.button} ${e.action} at (${event.x}, ${event.y})`); + } + } + break; + case 'ClientCutText': + message(`[${timeStr}] Clipboard: "${event.text}"`); + break; + // Skip SetPixelFormat, SetEncodings, FramebufferUpdateRequest - too noisy + } + }; player.start(realtime); } diff --git a/tests/playback.js b/tests/playback.js index 881a55bf..215ed7af 100644 --- a/tests/playback.js +++ b/tests/playback.js @@ -77,6 +77,119 @@ export default class RecordingPlayer { this._running = false; this.onfinish = () => {}; + this.onclientevent = () => {}; // Callback for client events + + this._lastButtonMask = 0; // Track previous button state for down/up detection + } + + // Decode client-to-server RFB message + _decodeClientMessage(data) { + if (data.length < 1) return null; + + const msgType = data[0]; + + switch (msgType) { + case 0: // SetPixelFormat + return { type: 'SetPixelFormat' }; + + case 2: // SetEncodings + if (data.length >= 4) { + const numEncodings = (data[2] << 8) | data[3]; + return { type: 'SetEncodings', count: numEncodings }; + } + return { type: 'SetEncodings' }; + + case 3: // FramebufferUpdateRequest + if (data.length >= 10) { + const incremental = data[1]; + const x = (data[2] << 8) | data[3]; + const y = (data[4] << 8) | data[5]; + const width = (data[6] << 8) | data[7]; + const height = (data[8] << 8) | data[9]; + return { + type: 'FramebufferUpdateRequest', + incremental: incremental === 1, + x, y, width, height + }; + } + return { type: 'FramebufferUpdateRequest' }; + + case 4: // KeyEvent + if (data.length >= 8) { + const down = data[1] === 1; + const keysym = (data[4] << 24) | (data[5] << 16) | (data[6] << 8) | data[7]; + // Try to convert keysym to character + let keyName = '0x' + keysym.toString(16); + if (keysym >= 0x20 && keysym <= 0x7e) { + keyName = String.fromCharCode(keysym); + } else if (keysym >= 0xff00) { + // Special keys + const specialKeys = { + 0xff08: 'BackSpace', 0xff09: 'Tab', 0xff0d: 'Return', + 0xff1b: 'Escape', 0xff50: 'Home', 0xff51: 'Left', + 0xff52: 'Up', 0xff53: 'Right', 0xff54: 'Down', + 0xff55: 'PageUp', 0xff56: 'PageDown', 0xff57: 'End', + 0xff63: 'Insert', 0xffff: 'Delete', + 0xffe1: 'Shift_L', 0xffe2: 'Shift_R', + 0xffe3: 'Control_L', 0xffe4: 'Control_R', + 0xffe9: 'Alt_L', 0xffea: 'Alt_R', + 0xffeb: 'Super_L', 0xffec: 'Super_R', + }; + keyName = specialKeys[keysym] || keyName; + } + return { type: 'KeyEvent', down, keysym, keyName }; + } + return { type: 'KeyEvent' }; + + case 5: // PointerEvent + if (data.length >= 6) { + const buttonMask = data[1]; + const x = (data[2] << 8) | data[3]; + const y = (data[4] << 8) | data[5]; + + // Detect button changes by comparing with previous state + const prevMask = this._lastButtonMask; + const pressed = buttonMask & ~prevMask; // Bits that are now 1 but were 0 + const released = prevMask & ~buttonMask; // Bits that are now 0 but were 1 + this._lastButtonMask = buttonMask; + + const events = []; + // Check each button for down/up + const buttonNames = ['left', 'middle', 'right', 'scrollUp', 'scrollDown']; + for (let i = 0; i < 5; i++) { + const bit = 1 << i; + if (pressed & bit) { + events.push({ button: buttonNames[i], action: 'down' }); + } + if (released & bit) { + events.push({ button: buttonNames[i], action: 'up' }); + } + } + + return { + type: 'PointerEvent', + x, y, + buttonMask, + events: events, // Array of {button, action} for changes + isMove: events.length === 0 + }; + } + return { type: 'PointerEvent' }; + + case 6: // ClientCutText + if (data.length >= 8) { + const length = (data[4] << 24) | (data[5] << 16) | (data[6] << 8) | data[7]; + let text = ''; + for (let i = 8; i < Math.min(8 + length, data.length); i++) { + text += String.fromCharCode(data[i]); + } + return { type: 'ClientCutText', text: text.substring(0, 50) + (length > 50 ? '...' : '') }; + } + return { type: 'ClientCutText' }; + + default: + return { type: 'Unknown', msgType }; + } } run(realtime, trafficManagement) { @@ -105,8 +218,13 @@ export default class RecordingPlayer { let frame = this._frames[this._frameIndex]; - // skip send frames + // Process and report client frames, then skip them while (this._frameIndex < this._frameLength && frame.fromClient) { + // Decode and report the client event + const decoded = this._decodeClientMessage(frame.data); + if (decoded) { + this.onclientevent(frame.timestamp, decoded); + } this._frameIndex++; frame = this._frames[this._frameIndex]; } diff --git a/utils/novnc_proxy b/utils/novnc_proxy index 6b55504a..8ad68620 100755 --- a/utils/novnc_proxy +++ b/utils/novnc_proxy @@ -34,12 +34,15 @@ usage() { echo " --heartbeat SEC send a ping to the client every SEC seconds" echo " --timeout SEC after SEC seconds exit when not connected" echo " --idle-timeout SEC server exits after SEC seconds if there are no" + echo " active connections" echo " " echo " --web-auth enable authentication" echo " --auth-plugin CLASS authentication plugin to use" echo " --auth-source ARG plugin configuration" echo " " - echo " active connections" + echo " --password PASS VNC password (passed to vnc.html URL)" + echo " --autoconnect auto-connect on page load" + echo " --autorecord auto-record on page load" echo " " exit 2 } @@ -65,6 +68,9 @@ WEBAUTH_ARG="" AUTHPLUGIN_ARG="" AUTHSOURCE_ARG="" FILEONLY_ARG="" +VNC_PASSWORD="" +AUTOCONNECT="" +AUTORECORD="" die() { @@ -103,6 +109,9 @@ while [ "$*" ]; do --web-auth) WEBAUTH_ARG="--web-auth" ;; --auth-plugin) AUTHPLUGIN_ARG="--auth-plugin ${OPTARG}"; shift ;; --auth-source) AUTHSOURCE_ARG="--auth-source ${OPTARG}"; shift ;; + --password) VNC_PASSWORD="${OPTARG}"; shift ;; + --autoconnect) AUTOCONNECT="1" ;; + --autorecord) AUTORECORD="1" ;; -h|--help) usage ;; -*) usage "Unknown chrooter option: ${param}" ;; *) break ;; @@ -223,10 +232,14 @@ if [ -z "$HOST" ]; then fi echo -e "\n\nNavigate to this URL:\n" +URL_PARAMS="host=${HOST}&port=${PORT}" +[ -n "$VNC_PASSWORD" ] && URL_PARAMS="${URL_PARAMS}&password=${VNC_PASSWORD}" +[ -n "$AUTOCONNECT" ] && URL_PARAMS="${URL_PARAMS}&autoconnect=1" +[ -n "$AUTORECORD" ] && URL_PARAMS="${URL_PARAMS}&autorecord=1" if [ "x$SSLONLY" == "x" ]; then - echo -e " http://${HOST}:${PORT}/vnc.html?host=${HOST}&port=${PORT}\n" + echo -e " http://${HOST}:${PORT}/vnc.html?${URL_PARAMS}\n" else - echo -e " https://${HOST}:${PORT}/vnc.html?host=${HOST}&port=${PORT}\n" + echo -e " https://${HOST}:${PORT}/vnc.html?${URL_PARAMS}\n" fi echo -e "Press Ctrl-C to exit\n\n" diff --git a/vnc.html b/vnc.html index 82cacd58..81b2909f 100644 --- a/vnc.html +++ b/vnc.html @@ -190,6 +190,24 @@ + + +
+
+
+ Recording +
+

Not recording

+

+

+ + + +
+
+