diff --git a/tests/perf/rfb_stream_bench.html b/tests/perf/rfb_stream_bench.html new file mode 100644 index 00000000..61291004 --- /dev/null +++ b/tests/perf/rfb_stream_bench.html @@ -0,0 +1,15 @@ + + + + + noVNC RFB Stream Benchmark + + + +
+

noVNC RFB Stream Benchmark

+

idle

+

+        
+ + diff --git a/tests/perf/rfb_stream_bench.js b/tests/perf/rfb_stream_bench.js new file mode 100644 index 00000000..e8b04b75 --- /dev/null +++ b/tests/perf/rfb_stream_bench.js @@ -0,0 +1,317 @@ +const statusEl = document.getElementById('status'); +const resultEl = document.getElementById('result'); + +const search = new URLSearchParams(window.location.search); +const scenario = search.get('scenario') || 'protocol-copyrect'; +const repoUrl = search.get('repo'); + +window.__BENCH_DONE = false; +window.__RESULT = null; +window.__ERROR = null; + +function setStatus(text) { + statusEl.textContent = text; + document.title = text; +} + +function setResult(result) { + window.__RESULT = result; + window.__BENCH_DONE = true; + setStatus('done'); + resultEl.textContent = JSON.stringify(result, null, 2); +} + +function setError(error) { + const rendered = error instanceof Error ? `${error.message}\n${error.stack || ''}` : String(error); + window.__ERROR = rendered; + window.__BENCH_DONE = true; + setStatus('error'); + resultEl.textContent = rendered; + console.error(error); +} + +function parseInteger(name, fallback) { + const raw = search.get(name); + if (raw === null || raw === '') { + return fallback; + } + + const value = Number(raw); + if (!Number.isInteger(value) || value < 0) { + throw new Error(`Invalid integer for "${name}": ${raw}`); + } + + return value; +} + +function ensureRepoUrl(rawRepoUrl) { + if (!rawRepoUrl) { + throw new Error('Missing "repo" query parameter'); + } + + const normalized = rawRepoUrl.endsWith('/') ? rawRepoUrl : `${rawRepoUrl}/`; + return new URL(normalized); +} + +function receive(ws, bytes) { + ws._receiveData(bytes); +} + +function dispatchMessage(ws, data) { + ws.onmessage(new MessageEvent('message', { data })); +} + +function createVersionBuffer() { + return new TextEncoder().encode('RFB 003.008\n'); +} + +function encodeName(name) { + return new TextEncoder().encode(name); +} + +function buildServerInit(width, height, name) { + const nameBytes = encodeName(name); + + const header = new Uint8Array(24); + const view = new DataView(header.buffer); + view.setUint16(0, width, false); + view.setUint16(2, height, false); + header[4] = 24; + header[5] = 24; + header[6] = 0; + header[7] = 1; + view.setUint16(8, 255, false); + view.setUint16(10, 255, false); + view.setUint16(12, 255, false); + header[14] = 16; + header[15] = 8; + header[16] = 0; + view.setUint32(20, nameBytes.length, false); + + return { header, nameBytes }; +} + +function handshake(ws, width, height, name) { + receive(ws, createVersionBuffer()); + receive(ws, new Uint8Array([1, 1])); + receive(ws, new Uint8Array([0, 0, 0, 0])); + + const init = buildServerInit(width, height, name); + receive(ws, init.header); + receive(ws, init.nameBytes); +} + +async function loadModules(repoBase) { + // util/browser.js has a top-level WebCodecs probe that can stall in + // headless Chrome. Remove the APIs before importing RFB so the module + // resolves quickly and deterministically in benchmark runs. + try { delete window.VideoDecoder; } catch {} + try { delete window.EncodedVideoChunk; } catch {} + + const [rfbModule, websocketModule] = await Promise.all([ + import(new URL('core/rfb.js', repoBase).href), + import(new URL('tests/fake.websocket.js', repoBase).href), + ]); + + return { + RFB: rfbModule.default, + FakeWebSocket: websocketModule.default, + }; +} + +function createHarness(RFB, FakeWebSocket) { + const mount = document.createElement('div'); + document.body.appendChild(mount); + + const ws = new FakeWebSocket(); + const rfb = new RFB(mount, ws); + ws._open(); + + return { mount, rfb, ws }; +} + +function destroyHarness(harness) { + try { + harness.rfb.disconnect(); + } catch {} + + if (harness.mount.parentNode !== null) { + harness.mount.parentNode.removeChild(harness.mount); + } +} + +function buildCopyRectBatch(rectsPerMessage) { + const bytesPerRect = 16; + const buffer = new ArrayBuffer(4 + (rectsPerMessage * bytesPerRect)); + const view = new DataView(buffer); + let offset = 0; + + view.setUint8(offset++, 0); + view.setUint8(offset++, 0); + view.setUint16(offset, rectsPerMessage, false); + offset += 2; + + for (let i = 0; i < rectsPerMessage; i++) { + view.setUint16(offset, 0, false); offset += 2; + view.setUint16(offset, 0, false); offset += 2; + view.setUint16(offset, 1, false); offset += 2; + view.setUint16(offset, 1, false); offset += 2; + view.setInt32(offset, 1, false); offset += 4; // CopyRect encoding + view.setUint16(offset, 0, false); offset += 2; + view.setUint16(offset, 0, false); offset += 2; + } + + return buffer; +} + +function createCopyRectMessages(messageCount, rectsPerMessage) { + const messages = []; + for (let i = 0; i < messageCount; i++) { + messages.push(buildCopyRectBatch(rectsPerMessage)); + } + return messages; +} + +function average(values) { + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +async function runSmokeRaw(RFB, FakeWebSocket) { + const start = performance.now(); + const harness = createHarness(RFB, FakeWebSocket); + + try { + handshake(harness.ws, 2, 2, 'smoke'); + + const update = new Uint8Array(4 + 12 + 4); + const view = new DataView(update.buffer); + let offset = 0; + + update[offset++] = 0; + update[offset++] = 0; + view.setUint16(offset, 1, false); offset += 2; + + view.setUint16(offset, 0, false); offset += 2; + view.setUint16(offset, 0, false); offset += 2; + view.setUint16(offset, 1, false); offset += 2; + view.setUint16(offset, 1, false); offset += 2; + view.setInt32(offset, 0, false); offset += 4; // Raw encoding + + update[offset++] = 0xff; + update[offset++] = 0x00; + update[offset++] = 0x00; + update[offset++] = 0x00; + + receive(harness.ws, update); + await new Promise((resolve) => window.setTimeout(resolve, 50)); + + const pixel = Array.from(harness.rfb.getImageData().data.slice(0, 4)); + const expectedPixel = [255, 0, 0, 255]; + const pass = harness.rfb._rfbConnectionState === 'connected' && + pixel.length === expectedPixel.length && + pixel.every((value, index) => value === expectedPixel[index]); + + return { + scenario: 'smoke-raw', + pass, + state: harness.rfb._rfbConnectionState, + pixel, + expectedPixel, + durationMs: performance.now() - start, + }; + } finally { + destroyHarness(harness); + } +} + +async function runProtocolCopyRect(RFB, FakeWebSocket) { + const iterations = parseInteger('iterations', 6); + const warmup = parseInteger('warmup', 2); + const messages = parseInteger('messages', 50); + const rects = parseInteger('rects', 5000); + const stubDisplay = search.get('stubDisplay') !== '0'; + const harness = createHarness(RFB, FakeWebSocket); + + try { + handshake(harness.ws, 4, 4, 'bench'); + harness.rfb._enabledContinuousUpdates = true; + + if (typeof harness.ws._getSentData === 'function') { + harness.ws._getSentData(); + } + + if (stubDisplay) { + harness.rfb._display.copyImage = () => {}; + harness.rfb._display.fillRect = () => {}; + harness.rfb._display.blitImage = () => {}; + harness.rfb._display.flip = () => {}; + } + + const payloads = createCopyRectMessages(messages, rects); + const feed = () => { + for (const payload of payloads) { + dispatchMessage(harness.ws, payload); + } + }; + + for (let i = 0; i < warmup; i++) { + feed(); + } + + const samples = []; + for (let i = 0; i < iterations; i++) { + const start = performance.now(); + feed(); + + if (typeof harness.rfb._display.flush === 'function' && harness.rfb._display.pending()) { + await harness.rfb._display.flush(); + } + + samples.push(performance.now() - start); + } + + const avg = average(samples); + const min = Math.min(...samples); + const max = Math.max(...samples); + const rectsPerIteration = messages * rects; + + return { + scenario: 'protocol-copyrect', + stubDisplay, + iterations, + warmup, + messages, + rectsPerMessage: rects, + rectsPerIteration, + avgMs: avg, + minMs: min, + maxMs: max, + rectsPerSecond: rectsPerIteration / (avg / 1000), + samplesMs: samples, + }; + } finally { + destroyHarness(harness); + } +} + +async function run() { + const repoBase = ensureRepoUrl(repoUrl); + setStatus(`loading ${scenario}`); + const { RFB, FakeWebSocket } = await loadModules(repoBase); + + if (scenario === 'smoke-raw') { + setStatus('running smoke-raw'); + setResult(await runSmokeRaw(RFB, FakeWebSocket)); + return; + } + + if (scenario === 'protocol-copyrect') { + setStatus('running protocol-copyrect'); + setResult(await runProtocolCopyRect(RFB, FakeWebSocket)); + return; + } + + throw new Error(`Unknown scenario "${scenario}"`); +} + +run().catch(setError); diff --git a/tests/perf/run_rfb_bench.mjs b/tests/perf/run_rfb_bench.mjs new file mode 100644 index 00000000..852d9fcd --- /dev/null +++ b/tests/perf/run_rfb_bench.mjs @@ -0,0 +1,489 @@ +#!/usr/bin/env node + +import { mkdtemp, rm } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import http from 'node:http'; +import net from 'node:net'; +import { spawn, spawnSync } from 'node:child_process'; +import { setTimeout as delay } from 'node:timers/promises'; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const pageUrl = pathToFileURL(path.join(scriptDir, 'rfb_stream_bench.html')).href; +const repoRoot = path.resolve(scriptDir, '..', '..'); + +const DEFAULT_SCENARIOS = [ + { + name: 'smoke-raw', + type: 'smoke', + scenario: 'smoke-raw', + }, + { + name: 'protocol-copyrect-parser', + type: 'compare', + scenario: 'protocol-copyrect', + params: { + iterations: 6, + warmup: 2, + messages: 50, + rects: 5000, + stubDisplay: 1, + }, + }, + { + name: 'protocol-copyrect-full', + type: 'compare', + scenario: 'protocol-copyrect', + params: { + iterations: 5, + warmup: 2, + messages: 20, + rects: 2000, + stubDisplay: 0, + }, + }, +]; + +function printHelp() { + console.log(`Usage: node tests/perf/run_rfb_bench.mjs [options] + +Options: + --candidate Repo checkout to benchmark. Defaults to the current checkout. + --baseline Baseline repo checkout. If omitted, a temporary worktree is created. + --baseline-ref Git ref to use for the temporary baseline worktree. + --chrome Chrome/Chromium executable to launch. + --json Print JSON only. + --keep-worktree Keep the temporary baseline worktree on disk. + --help Show this message. +`); +} + +function parseArgs(argv) { + const options = { + candidate: repoRoot, + baseline: null, + baselineRef: null, + chrome: null, + json: false, + keepWorktree: false, + }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + switch (arg) { + case '--candidate': + options.candidate = argv[++i]; + break; + case '--baseline': + options.baseline = argv[++i]; + break; + case '--baseline-ref': + options.baselineRef = argv[++i]; + break; + case '--chrome': + options.chrome = argv[++i]; + break; + case '--json': + options.json = true; + break; + case '--keep-worktree': + options.keepWorktree = true; + break; + case '--help': + options.help = true; + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + + return options; +} + +function runCommand(command, args, cwd) { + const result = spawnSync(command, args, { + cwd, + encoding: 'utf8', + }); + + if (result.status !== 0) { + throw new Error([ + `Command failed: ${command} ${args.join(' ')}`, + result.stdout.trim(), + result.stderr.trim(), + ].filter(Boolean).join('\n')); + } + + return result.stdout.trim(); +} + +function resolveChromeExecutable(explicitChrome) { + if (explicitChrome) { + return explicitChrome; + } + + if (process.env.CHROME_BIN) { + return process.env.CHROME_BIN; + } + + const candidates = [ + 'google-chrome', + 'google-chrome-stable', + 'chromium', + 'chromium-browser', + ]; + + for (const candidate of candidates) { + const result = spawnSync('bash', ['-lc', `command -v ${candidate}`], { + encoding: 'utf8', + }); + + if (result.status === 0) { + return result.stdout.trim(); + } + } + + throw new Error('Unable to locate a Chrome/Chromium executable'); +} + +function resolveDefaultBaselineRef(candidateRoot) { + const refs = ['origin/master', 'master', 'origin/main', 'main']; + + for (const ref of refs) { + const result = spawnSync('git', ['-C', candidateRoot, 'rev-parse', '--verify', '--quiet', `${ref}^{commit}`], { + encoding: 'utf8', + }); + + if (result.status === 0) { + return ref; + } + } + + throw new Error('Unable to resolve a default baseline ref (tried origin/master, master, origin/main, main)'); +} + +function dirToFileUrl(dirPath) { + const href = pathToFileURL(path.resolve(dirPath)).href; + return href.endsWith('/') ? href : `${href}/`; +} + +async function allocatePort() { + return await new Promise((resolve, reject) => { + const server = net.createServer(); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + server.close((error) => { + if (error) { + reject(error); + return; + } + + resolve(address.port); + }); + }); + server.on('error', reject); + }); +} + +async function httpGetJson(url) { + return await new Promise((resolve, reject) => { + const request = http.get(url, (response) => { + let data = ''; + response.setEncoding('utf8'); + response.on('data', (chunk) => { data += chunk; }); + response.on('end', () => { + try { + resolve(JSON.parse(data)); + } catch (error) { + reject(error); + } + }); + }); + + request.on('error', reject); + }); +} + +async function waitForDevtools(port, timeoutMs) { + const deadline = Date.now() + timeoutMs; + + while (Date.now() < deadline) { + try { + await httpGetJson(`http://127.0.0.1:${port}/json/version`); + return; + } catch (error) { + await delay(100); + } + } + + throw new Error(`Timed out waiting for Chrome DevTools on port ${port}`); +} + +async function getPageTarget(port, expectedUrl, timeoutMs) { + const deadline = Date.now() + timeoutMs; + + while (Date.now() < deadline) { + const pages = await httpGetJson(`http://127.0.0.1:${port}/json/list`); + const page = pages.find((entry) => entry.type === 'page' && entry.url === expectedUrl); + + if (page) { + return page; + } + + await delay(100); + } + + throw new Error(`Timed out waiting for page target: ${expectedUrl}`); +} + +async function cdpEvaluate(page, expression) { + return await new Promise((resolve, reject) => { + const ws = new WebSocket(page.webSocketDebuggerUrl); + + ws.addEventListener('open', () => { + ws.send(JSON.stringify({ + id: 1, + method: 'Runtime.evaluate', + params: { + expression, + returnByValue: true, + }, + })); + }); + + ws.addEventListener('message', (event) => { + try { + resolve(JSON.parse(event.data.toString())); + } catch (error) { + reject(error); + } finally { + ws.close(); + } + }); + + ws.addEventListener('error', reject); + }); +} + +async function killChrome(child) { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + + const exited = new Promise((resolve) => child.once('exit', resolve)); + child.kill('SIGTERM'); + await Promise.race([exited, delay(5000)]); + + if (child.exitCode === null && child.signalCode === null) { + child.kill('SIGKILL'); + await Promise.race([exited, delay(5000)]); + } +} + +function createScenarioUrl(repoPath, scenario) { + const params = new URLSearchParams(); + params.set('repo', dirToFileUrl(repoPath)); + params.set('scenario', scenario.scenario); + + if (scenario.params) { + for (const [key, value] of Object.entries(scenario.params)) { + params.set(key, String(value)); + } + } + + return `${pageUrl}?${params.toString()}`; +} + +async function runScenario(chromeExecutable, repoPath, scenario) { + const port = await allocatePort(); + const userDataDir = await mkdtemp(path.join(os.tmpdir(), 'novnc-rfb-bench-chrome-')); + const logs = []; + const scenarioUrl = createScenarioUrl(repoPath, scenario); + const child = spawn(chromeExecutable, [ + '--headless', + '--no-sandbox', + '--disable-gpu', + '--disable-extensions', + '--disable-background-networking', + '--disable-component-update', + '--disable-default-apps', + '--disable-sync', + '--incognito', + '--no-default-browser-check', + '--no-first-run', + `--remote-debugging-port=${port}`, + `--user-data-dir=${userDataDir}`, + '--allow-file-access-from-files', + scenarioUrl, + ], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { logs.push(chunk.trim()); }); + child.stderr.on('data', (chunk) => { logs.push(chunk.trim()); }); + + try { + await waitForDevtools(port, 15000); + const page = await getPageTarget(port, scenarioUrl, 15000); + const deadline = Date.now() + 120000; + + while (Date.now() < deadline) { + const response = await cdpEvaluate(page, `(() => ({ + done: window.__BENCH_DONE === true, + error: window.__ERROR || null, + result: window.__RESULT || null, + status: document.getElementById('status')?.textContent || document.title, + }))()`); + + const state = response.result.result.value; + + if (state.done) { + if (state.error) { + throw new Error(`Scenario ${scenario.name} failed:\n${state.error}`); + } + + return state.result; + } + + await delay(200); + } + + throw new Error(`Scenario ${scenario.name} timed out`); + } catch (error) { + const logTail = logs.filter(Boolean).slice(-20).join('\n'); + if (logTail) { + error.message = `${error.message}\n\nChrome log tail:\n${logTail}`; + } + throw error; + } finally { + await killChrome(child); + await rm(userDataDir, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 100, + }); + } +} + +function formatMs(value) { + return `${value.toFixed(2)} ms`; +} + +function formatPct(value) { + return `${value.toFixed(1)}%`; +} + +function formatDeltaSentence(deltaPct) { + if (deltaPct >= 0) { + return `${formatPct(deltaPct)} faster on the candidate`; + } + + return `${formatPct(Math.abs(deltaPct))} slower on the candidate`; +} + +function summarizeComparison(baseline, candidate) { + const deltaPct = ((baseline.avgMs - candidate.avgMs) / baseline.avgMs) * 100; + return { + baseline, + candidate, + deltaPct, + }; +} + +function buildMarkdown(summary) { + const lines = [ + '## Browser-level smoke + protocol-stream benchmark', + '', + `- Smoke test (\`smoke-raw\`): baseline ${summary.smoke.baseline.pass ? 'pass' : 'fail'}, candidate ${summary.smoke.candidate.pass ? 'pass' : 'fail'}, candidate pixel ${JSON.stringify(summary.smoke.candidate.pixel)}.`, + `- Parser-focused protocol benchmark (\`CopyRect\`, display stubbed): baseline ${formatMs(summary.parser.baseline.avgMs)}, candidate ${formatMs(summary.parser.candidate.avgMs)}, ${formatDeltaSentence(summary.parser.deltaPct)}.`, + `- Full-pipeline protocol benchmark (\`CopyRect\`, display active): baseline ${formatMs(summary.full.baseline.avgMs)}, candidate ${formatMs(summary.full.candidate.avgMs)}, delta ${formatPct(summary.full.deltaPct)}.`, + '', + 'The parser-focused benchmark runs actual noVNC `RFB/Websock` protocol parsing on complete `FramebufferUpdate` messages and stubs the display layer so the measurement stays attributable to the receive path. When display work is included, the signal is largely drowned out by rendering cost, so the end-to-end delta is not a good attribution tool for this specific change.', + ]; + + return lines.join('\n'); +} + +async function createBaselineWorktree(candidateRoot, baselineRef) { + const tempRoot = await mkdtemp(path.join(os.tmpdir(), 'novnc-rfb-bench-baseline-')); + const worktreePath = path.join(tempRoot, 'checkout'); + runCommand('git', ['-C', candidateRoot, 'worktree', 'add', '--detach', worktreePath, baselineRef], candidateRoot); + return { tempRoot, worktreePath }; +} + +async function removeBaselineWorktree(candidateRoot, worktree) { + try { + runCommand('git', ['-C', candidateRoot, 'worktree', 'remove', '--force', worktree.worktreePath], candidateRoot); + } finally { + await rm(worktree.tempRoot, { recursive: true, force: true }); + } +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + printHelp(); + return; + } + + const chromeExecutable = resolveChromeExecutable(options.chrome); + const candidateRoot = path.resolve(options.candidate); + const baselineRef = options.baseline ? null : (options.baselineRef || resolveDefaultBaselineRef(candidateRoot)); + + let baselineRoot = options.baseline ? path.resolve(options.baseline) : null; + let worktree = null; + + if (!baselineRoot) { + worktree = await createBaselineWorktree(candidateRoot, baselineRef); + baselineRoot = worktree.worktreePath; + } + + try { + const smokeScenario = DEFAULT_SCENARIOS.find((scenario) => scenario.name === 'smoke-raw'); + const parserScenario = DEFAULT_SCENARIOS.find((scenario) => scenario.name === 'protocol-copyrect-parser'); + const fullScenario = DEFAULT_SCENARIOS.find((scenario) => scenario.name === 'protocol-copyrect-full'); + + const smokeBaseline = await runScenario(chromeExecutable, baselineRoot, smokeScenario); + const smokeCandidate = await runScenario(chromeExecutable, candidateRoot, smokeScenario); + const parserBaseline = await runScenario(chromeExecutable, baselineRoot, parserScenario); + const parserCandidate = await runScenario(chromeExecutable, candidateRoot, parserScenario); + const fullBaseline = await runScenario(chromeExecutable, baselineRoot, fullScenario); + const fullCandidate = await runScenario(chromeExecutable, candidateRoot, fullScenario); + + const summary = { + chromeExecutable, + candidateRoot, + baselineRoot, + baselineRef, + smoke: { + baseline: smokeBaseline, + candidate: smokeCandidate, + }, + parser: summarizeComparison(parserBaseline, parserCandidate), + full: summarizeComparison(fullBaseline, fullCandidate), + }; + + if (options.json) { + console.log(JSON.stringify(summary, null, 2)); + return; + } + + console.log(JSON.stringify(summary, null, 2)); + console.log('\nSuggested PR summary:\n'); + console.log(buildMarkdown(summary)); + } finally { + if (worktree && !options.keepWorktree) { + await removeBaselineWorktree(candidateRoot, worktree); + } + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)); + process.exitCode = 1; +});