#!/usr/bin/env node /** * perf/baseline.mjs * * Performance baseline harness for ytplayer PWA. * Drives the real Bun server and runs Playwright (Chromium with CDP throttling + CPU 4x; * WebKit with a local throttling & stalling proxy). * * Scenarios: * - cold: fresh context, network download, SW install, boot, media playback check * - warm: re-navigate with active SW and primed HTTP cache * - offline: context.setOffline(true), reload under SW cache * - update-js: update N->N+1 (1-line comment appended to app.js) -> wire bytes + time until new SW ready * - update-css: update N->N+1 (1-line rule appended to theme-glass.css) -> wire bytes + time until new SW ready * * Usage: * node perf/baseline.mjs [options] * Options: * --runs Runs per scenario for median calculation (default: 5) * --browser Browser(s) to test (default: all) * --profile Network profile (default: lte) * --scenario Specific scenario or all (default: all) * --out Output JSON path * --compare Compare results against a previous baseline JSON */ import assert from 'node:assert/strict'; import { assertChangedAssets, assertWarmAssets, waitForIdleAssets } from './measurement.mjs'; import { createHash } from 'node:crypto'; import { spawn, spawnSync, execFileSync } from 'node:child_process'; import { existsSync, mkdirSync, rmSync, cpSync, appendFileSync, statSync, writeFileSync, readFileSync, symlinkSync, unlinkSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { createThrottleProxy } from './proxy.mjs'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const REPO_ROOT = path.resolve(__dirname, '..'); // Dynamically resolve playwright from node_modules let pw; try { pw = await import('playwright'); } catch { pw = await import(path.join(REPO_ROOT, 'node_modules', 'playwright', 'index.mjs')); } import { createRequire } from 'node:module'; const req = createRequire(import.meta.url); const { createClient } = req(path.join(REPO_ROOT, 'server', 'node_modules', '@libsql/client')); const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); function parseArgs() { const args = process.argv.slice(2); const options = { runs: 5, browser: 'all', // 'chromium', 'webkit', or 'all' profile: 'lte', // 'lte', 'lossy', or 'all' scenario: 'all', // 'cold', 'warm', 'offline', 'update-js', 'update-css', or 'all' out: null, compare: null, frontendCommit: null, }; for (let i = 0; i < args.length; i++) { const a = args[i]; if (a === '--runs' && i + 1 < args.length) options.runs = parseInt(args[++i], 10); else if (a === '--browser' && i + 1 < args.length) options.browser = args[++i]; else if (a === '--profile' && i + 1 < args.length) options.profile = args[++i]; else if (a === '--scenario' && i + 1 < args.length) options.scenario = args[++i]; else if (a === '--out' && i + 1 < args.length) options.out = args[++i]; else if (a === '--frontend-commit' && i + 1 < args.length) options.frontendCommit = args[++i]; else if (a === '--compare' && i + 1 < args.length) options.compare = args[++i]; } return options; } function median(values) { const nums = values.filter((v) => typeof v === 'number' && !Number.isNaN(v)).sort((a, b) => a - b); if (!nums.length) return null; const mid = Math.floor(nums.length / 2); return nums.length % 2 !== 0 ? nums[mid] : Math.round(((nums[mid - 1] + nums[mid]) / 2) * 10) / 10; } function summarizeList(values) { const valid = values.filter((v) => v !== null && v !== undefined && !Number.isNaN(v)); if (!valid.length) return { median: null, min: null, max: null, values }; return { median: median(valid), min: Math.min(...valid), max: Math.max(...valid), values, }; } /** * Setup deterministic local media fixture */ function prepareMediaFiles(mediaDir) { mkdirSync(mediaDir, { recursive: true }); const mp4 = path.join(mediaDir, 'benchmedia01.1.mp4'); const m4a = path.join(mediaDir, 'benchmedia01.1.m4a'); if (!existsSync(mp4)) { spawnSync('ffmpeg', [ '-y', '-f', 'lavfi', '-i', 'testsrc=size=640x360:rate=30:duration=5', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=5', '-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-movflags', '+faststart', '-shortest', mp4, ]); } if (!existsSync(m4a)) { spawnSync('ffmpeg', [ '-y', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=5', '-c:a', 'aac', '-shortest', m4a, ]); } return { mp4, m4a }; } async function seedDatabase(dbPath, mediaFiles) { const db = createClient({ url: 'file:' + dbPath }); const mp4Size = statSync(mediaFiles.mp4).size; const m4aSize = statSync(mediaFiles.m4a).size; await db.execute({ sql: `INSERT OR REPLACE INTO media_cache ( video_id, status, gen, size, height, vcodec, acodec, duration, meta, last_access ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, args: [ 'benchmedia01', 'ready', 1, mp4Size + m4aSize, 360, 'h264', 'aac', 5.0, JSON.stringify({ title: 'Baseline Deterministic Clip', channel: 'Perf Benchmark' }), Date.now(), ], }); } /** * Bun Server Manager */ class ServerManager { constructor(scratchDir, publicDir) { this.scratchDir = scratchDir; this.publicDir = publicDir; this.dataDir = path.join(scratchDir, 'data'); this.mediaDir = path.join(this.dataDir, 'media'); this.dbPath = path.join(this.dataDir, 'ytplayer.db'); this.serverProcess = null; this.port = 0; } async start() { mkdirSync(this.dataDir, { recursive: true }); const fixtureDir = path.join(REPO_ROOT, 'perf', 'fixtures', 'media'); const mediaFiles = prepareMediaFiles(fixtureDir); // Create a dedicated server working dir with public symlink const srvCwd = path.join(this.scratchDir, 'srv'); mkdirSync(srvCwd, { recursive: true }); const publicLink = path.join(srvCwd, 'public'); if (existsSync(publicLink)) unlinkSync(publicLink); symlinkSync(this.publicDir, publicLink); // Pick random available ephemeral port once and reuse across restarts this.port = this.port || (4300 + Math.floor(Math.random() * 2000)); const env = { ...process.env, PORT: String(this.port), DATA_DIR: this.dataDir, MEDIA_DIR: this.mediaDir, DB_PATH: this.dbPath, }; const serverScript = path.join(REPO_ROOT, 'server', 'server.js'); this.serverProcess = spawn('bun', [serverScript], { cwd: srvCwd, env, stdio: 'ignore', }); // Wait for server readiness let ready = false; for (let i = 0; i < 50; i++) { try { const res = await fetch(`http://127.0.0.1:${this.port}/api/version`); if (res.ok) { ready = true; break; } } catch {} await sleep(100); } if (!ready) { throw new Error(`Server failed to start on port ${this.port}`); } // Copy media files to mediaDir and seed media table AFTER server startup sweep mkdirSync(this.mediaDir, { recursive: true }); const targetMp4 = path.join(this.mediaDir, 'benchmedia01.1.mp4'); const targetM4a = path.join(this.mediaDir, 'benchmedia01.1.m4a'); cpSync(mediaFiles.mp4, targetMp4); cpSync(mediaFiles.m4a, targetM4a); await seedDatabase(this.dbPath, { mp4: targetMp4, m4a: targetM4a }); } async stop() { if (this.serverProcess) { const proc = this.serverProcess; this.serverProcess = null; await new Promise((resolve) => { if (proc.exitCode !== null) return resolve(); proc.once('exit', () => resolve()); proc.kill('SIGKILL'); setTimeout(resolve, 2000); }); await sleep(200); } } async restartWithPublic(newPublicDir) { await this.stop(); this.publicDir = newPublicDir; await this.start(); } } /** * Configure page for metrics collection */ async function setupPageMetrics(page) { await page.addInitScript(() => { window.__bootDone = null; window.__fcp = null; window.__lcp = null; window.__longTasksTotal = 0; window.__longTasks = []; // Observe boot completion landmark (#cards view set by renderList()) const markBoot = () => { const cards = document.getElementById('cards'); if (cards && cards.dataset && cards.dataset.view) { if (!window.__bootDone) window.__bootDone = performance.now(); return true; } return false; }; const t = setInterval(() => { if (markBoot()) clearInterval(t); }, 5); // Observe Long Tasks API (supported on Chromium) try { new PerformanceObserver((list) => { for (const entry of list.getEntries()) { window.__longTasksTotal += entry.duration; window.__longTasks.push({start:Math.round(entry.startTime),duration:Math.round(entry.duration)}); } }).observe({ type: 'longtask', buffered: true }); } catch {} // Observe LCP try { new PerformanceObserver((list) => { const entries = list.getEntries(); if (entries.length > 0) { window.__lcp = entries[entries.length - 1].startTime; } }).observe({ type: 'largest-contentful-paint', buffered: true }); } catch {} }); } /** * Run Scenario: Cold Load */ async function runColdScenario({ browserType, server, proxy, profile }) { const isWebKit = browserType.name() === 'webkit'; const launchArgs = isWebKit ? [] : ['--autoplay-policy=no-user-gesture-required']; const browser = await browserType.launch({ headless: true, args: launchArgs, }); const contextOptions = { proxy: { server: `http://127.0.0.1:${proxy.port}` } }; const context = await browser.newContext(contextOptions); const page = await context.newPage(); // If Chromium, configure CDP Network conditions and 4x CPU throttle if (!isWebKit) { const cdp = await context.newCDPSession(page); await cdp.send('Network.enable'); const isLossy = profile === 'lossy'; await cdp.send('Network.emulateNetworkConditions', { offline: false, latency: isLossy ? 250 : 150, downloadThroughput: isLossy ? (1.0 * 1000 * 1000) / 8 : (1.6 * 1000 * 1000) / 8, uploadThroughput: (750 * 1000) / 8, connectionType: 'cellular4g', }); await cdp.send('Emulation.setCPUThrottlingRate', { rate: 4 }); } await setupPageMetrics(page); proxy.resetStats(); proxy.startTracking(); const swInstallStart = Date.now(); const navUrl = `http://127.0.0.1:${server.port}/?v=benchmedia01`; await page.goto(navUrl); // Wait for boot landmark await page.waitForFunction(() => window.__bootDone !== null, { timeout: 15000 }); // Check deterministic media playing-ready let mediaReadyMs = null; let autoplayBlocked = false; try { const mediaCheck = await page.waitForFunction(() => { const v = document.querySelector('video') || document.querySelector('audio'); if (!v) return false; return v.readyState >= 3 || (!v.paused && v.currentTime > 0); }, { timeout: 8000 }); if (mediaCheck) { const state = await page.evaluate(() => { const v = document.querySelector('video') || document.querySelector('audio'); return { paused: v.paused, readyState: v.readyState }; }); autoplayBlocked = state.paused; mediaReadyMs = await page.evaluate(() => performance.now()); } } catch { autoplayBlocked = true; } // Wait for initial Service Worker activation await page.evaluate(async () => { const reg = await navigator.serviceWorker.ready; if (reg.active && reg.active.state === 'activated') return; await new Promise((resolve) => { const w = reg.active || reg.installing; if (w.state === 'activated') return resolve(); w.addEventListener('statechange', () => { if (w.state === 'activated') resolve(); }); }); }); const swInstallDuration = Date.now() - swInstallStart; proxy.stopTracking(); const stats = proxy.getStats(); const pageMetrics = await page.evaluate(() => { const nav = performance.getEntriesByType('navigation')[0]; const paint = performance.getEntriesByType('paint'); const fcp = paint.find((p) => p.name === 'first-contentful-paint'); return { initialJsCss: performance.getEntriesByType('resource').filter(entry=>/\.(?:js|css)(?:[?#]|$)/.test(entry.name)&&entry.startTime <= nav.domContentLoadedEventEnd&&['script','link','css'].includes(entry.initiatorType)).reduce((sum,entry)=>sum+entry.encodedBodySize,0), initialAssetRequests: performance.getEntriesByType('resource').filter(entry=>/\.(?:js|css)(?:[?#]|$)/.test(entry.name)&&entry.startTime <= nav.domContentLoadedEventEnd&&['script','link','css'].includes(entry.initiatorType)).map(entry=>({url:entry.name,bytes:entry.encodedBodySize})), fcp: fcp ? Math.round(fcp.startTime) : null, lcp: window.__lcp ? Math.round(window.__lcp) : null, domContentLoaded: nav ? Math.round(nav.domContentLoadedEventEnd - nav.startTime) : null, bootDone: window.__bootDone ? Math.round(window.__bootDone) : null, longTaskTotal: Math.round(window.__longTasksTotal || 0), longTasks: window.__longTasks || [], }; }); const storage = await page.evaluate(async () => { if (navigator.storage && navigator.storage.estimate) { const est = await navigator.storage.estimate(); return { usage: est.usage || 0, quota: est.quota || 0 }; } return { usage: null, quota: null }; }); await browser.close(); return { requestCount: stats.requestCount, wireBytes: stats.totalWireBytes, wireBytesByType: stats.bytesByType, fcp: pageMetrics.fcp, lcp: pageMetrics.lcp, domContentLoaded: pageMetrics.domContentLoaded, bootDone: pageMetrics.bootDone, longTaskTotal: pageMetrics.longTaskTotal, longTasks: pageMetrics.longTasks, requests: stats.requests, initialJsCss: pageMetrics.initialJsCss, initialAssetRequests: pageMetrics.initialAssetRequests, swInstallDuration, swInstallBytes: stats.totalWireBytes, storageEstimate: storage, mediaReady: mediaReadyMs ? Math.round(mediaReadyMs) : null, autoplayBlocked, }; } /** * Run Scenario: Warm Load (re-navigation with active SW) */ async function runWarmScenario({ browserType, server, proxy, profile }) { const isWebKit = browserType.name() === 'webkit'; const launchArgs = isWebKit ? [] : ['--autoplay-policy=no-user-gesture-required']; const browser = await browserType.launch({ headless: true, args: launchArgs, }); const contextOptions = { proxy: { server: `http://127.0.0.1:${proxy.port}` } }; const context = await browser.newContext(contextOptions); const page = await context.newPage(); if (!isWebKit) { const cdp = await context.newCDPSession(page); await cdp.send('Network.enable'); const isLossy = profile === 'lossy'; await cdp.send('Network.emulateNetworkConditions', { offline: false, latency: isLossy ? 250 : 150, downloadThroughput: isLossy ? (1.0 * 1000 * 1000) / 8 : (1.6 * 1000 * 1000) / 8, uploadThroughput: (750 * 1000) / 8, connectionType: 'cellular4g', }); await cdp.send('Emulation.setCPUThrottlingRate', { rate: 4 }); } await setupPageMetrics(page); const navUrl = `http://127.0.0.1:${server.port}/`; // Prime context with cold load await page.goto(navUrl); await page.waitForFunction(() => window.__bootDone !== null, { timeout: 15000 }); await page.evaluate(async () => { const reg = await navigator.serviceWorker.ready; if (reg.active && reg.active.state === 'activated') return; await new Promise((resolve) => { const w = reg.active || reg.installing; if (w.state === 'activated') return resolve(); w.addEventListener('statechange', () => { if (w.state === 'activated') resolve(); }); }); }); await waitForIdleAssets(page); // Now measure warm re-navigation proxy.resetStats(); proxy.startTracking(); await page.goto(navUrl); await page.waitForFunction(() => window.__bootDone !== null, { timeout: 15000 }); proxy.stopTracking(); const stats = proxy.getStats(); assertWarmAssets(stats.requests); const pageMetrics = await page.evaluate(() => { const nav = performance.getEntriesByType('navigation')[0]; const paint = performance.getEntriesByType('paint'); const fcp = paint.find((p) => p.name === 'first-contentful-paint'); return { initialJsCss: performance.getEntriesByType('resource').filter(entry=>/\.(?:js|css)(?:[?#]|$)/.test(entry.name)&&entry.startTime <= nav.domContentLoadedEventEnd&&['script','link','css'].includes(entry.initiatorType)).reduce((sum,entry)=>sum+entry.encodedBodySize,0), initialAssetRequests: performance.getEntriesByType('resource').filter(entry=>/\.(?:js|css)(?:[?#]|$)/.test(entry.name)&&entry.startTime <= nav.domContentLoadedEventEnd&&['script','link','css'].includes(entry.initiatorType)).map(entry=>({url:entry.name,bytes:entry.encodedBodySize})), fcp: fcp ? Math.round(fcp.startTime) : null, lcp: window.__lcp ? Math.round(window.__lcp) : null, domContentLoaded: nav ? Math.round(nav.domContentLoadedEventEnd - nav.startTime) : null, bootDone: window.__bootDone ? Math.round(window.__bootDone) : null, longTaskTotal: Math.round(window.__longTasksTotal || 0), longTasks: window.__longTasks || [], }; }); await browser.close(); return { requestCount: stats.requestCount, wireBytes: stats.totalWireBytes, wireBytesByType: stats.bytesByType, fcp: pageMetrics.fcp, lcp: pageMetrics.lcp, domContentLoaded: pageMetrics.domContentLoaded, bootDone: pageMetrics.bootDone, longTaskTotal: pageMetrics.longTaskTotal, longTasks: pageMetrics.longTasks, }; } /** * Run Scenario: Offline Reload under SW */ async function runOfflineScenario({ browserType, server, proxy, profile }) { const browser = await browserType.launch({ headless: true }); const isWebKit = browserType.name() === 'webkit'; const contextOptions = { proxy: { server: `http://127.0.0.1:${proxy.port}` } }; const context = await browser.newContext(contextOptions); const page = await context.newPage(); await setupPageMetrics(page); const navUrl = `http://127.0.0.1:${server.port}/`; // Prime SW await page.goto(navUrl); await page.waitForFunction(() => window.__bootDone !== null, { timeout: 15000 }); await page.evaluate(async () => { const reg = await navigator.serviceWorker.ready; if (reg.active && reg.active.state === 'activated') return; await new Promise((resolve) => { const w = reg.active || reg.installing; if (w.state === 'activated') return resolve(); w.addEventListener('statechange', () => { if (w.state === 'activated') resolve(); }); }); }); // Switch to offline mode: // On Chromium, context.setOffline(true) works. // On Linux WebKit, Playwright context.setOffline throws an internal browser error, // so proxy.setOffline(true) cuts upstream network to simulate offline truthfully. if (!isWebKit) { await context.setOffline(true); } proxy.setOffline(true); proxy.resetStats(); proxy.startTracking(); const t0 = Date.now(); await page.reload(); await page.waitForFunction(() => window.__bootDone !== null, { timeout: 15000 }); const reloadDuration = Date.now() - t0; proxy.stopTracking(); proxy.setOffline(false); const stats = proxy.getStats(); const pageMetrics = await page.evaluate(() => { const nav = performance.getEntriesByType('navigation')[0]; const paint = performance.getEntriesByType('paint'); const fcp = paint.find((p) => p.name === 'first-contentful-paint'); return { initialJsCss: performance.getEntriesByType('resource').filter(entry=>/\.(?:js|css)(?:[?#]|$)/.test(entry.name)&&entry.startTime <= nav.domContentLoadedEventEnd&&['script','link','css'].includes(entry.initiatorType)).reduce((sum,entry)=>sum+entry.encodedBodySize,0), initialAssetRequests: performance.getEntriesByType('resource').filter(entry=>/\.(?:js|css)(?:[?#]|$)/.test(entry.name)&&entry.startTime <= nav.domContentLoadedEventEnd&&['script','link','css'].includes(entry.initiatorType)).map(entry=>({url:entry.name,bytes:entry.encodedBodySize})), fcp: fcp ? Math.round(fcp.startTime) : null, lcp: window.__lcp ? Math.round(window.__lcp) : null, domContentLoaded: nav ? Math.round(nav.domContentLoadedEventEnd - nav.startTime) : null, bootDone: window.__bootDone ? Math.round(window.__bootDone) : null, }; }); await browser.close(); return { requestCount: stats.requestCount, wireBytes: stats.totalWireBytes, fcp: pageMetrics.fcp, lcp: pageMetrics.lcp, domContentLoaded: pageMetrics.domContentLoaded, bootDone: pageMetrics.bootDone, reloadDuration, }; } /** * Run Scenario: Update N -> N+1 */ async function runUpdateScenario({ browserType, server, proxy, profile, updateKind, trees }) { const browser = await browserType.launch({ headless: true }); const isWebKit = browserType.name() === 'webkit'; const contextOptions = { proxy: { server: `http://127.0.0.1:${proxy.port}` } }; const context = await browser.newContext(contextOptions); const page = await context.newPage(); if (!isWebKit) { const cdp = await context.newCDPSession(page); await cdp.send('Network.enable'); const isLossy = profile === 'lossy'; await cdp.send('Network.emulateNetworkConditions', { offline: false, latency: isLossy ? 250 : 150, downloadThroughput: isLossy ? (1.0 * 1000 * 1000) / 8 : (1.6 * 1000 * 1000) / 8, uploadThroughput: (750 * 1000) / 8, connectionType: 'cellular4g', }); } // Start on Tree N (base) await server.restartWithPublic(trees.treeBase); const navUrl = `http://127.0.0.1:${server.port}/`; await page.goto(navUrl); // Wait for steady state: active worker present, no waiting worker while (true) { const ready = await page.evaluate(async () => { const reg = await navigator.serviceWorker.getRegistration(); return reg?.active?.state === 'activated' && !reg?.installing && !reg?.waiting; }); if (ready) break; await sleep(50); } // Phase 3 activation precedes idle feature caching. Measure updates from a // genuinely warm build, rather than counting unfinished first-install bytes. await waitForIdleAssets(page); // Switch server to Tree N+1 (app.js or theme-glass.css modification) const targetTree = updateKind === 'feature' ? trees.treeUpdateFeature : updateKind === 'js' ? trees.treeUpdateJs : trees.treeUpdateCss; await server.restartWithPublic(targetTree); // Measure wire bytes and duration from reg.update() until new SW is waiting/installed proxy.resetStats(); proxy.startTracking(); const t0 = Date.now(); await page.evaluate(() => navigator.serviceWorker.getRegistration().then((r) => r.update())); // Wait until new worker reaches waiting state while (true) { const waiting = await page.evaluate(async () => { const reg = await navigator.serviceWorker.getRegistration(); return reg?.waiting?.state === 'installed'; }); if (waiting) break; await sleep(50); } const readyDuration = Date.now() - t0; const readyWireBytes = proxy.getStats().totalWireBytes; if (updateKind === 'feature' || updateKind === 'css') { const changedPath=updateKind==='feature'?'/presenter.js':'/theme-glass.css'; // Readiness excludes background chunks. Include activation and the changed // feature's actual idle transfer rather than reporting readiness as its cost. const expected = createHash('sha256').update(readFileSync(path.join(targetTree,changedPath.slice(1)))).digest('hex').slice(0,10); await page.evaluate(async()=>{const r=await navigator.serviceWorker.getRegistration();r.waiting.postMessage({type:'SKIP_WAITING'});}); const end=Date.now()+90000; while(true) { const complete=await page.evaluate(async ({h,p})=>{ const r=await navigator.serviceWorker.getRegistration(); if(r.waiting || r.installing)return false; r.active?.postMessage({type:'WARM_ASSETS',saveData:false}); const c=await caches.open('ytplayer-assets'),entry=await c.match(p+'?v='+h); return entry?.headers.get('X-Asset-Hash')===h; },{h:expected,p:changedPath}); if(complete)break; if(Date.now()>end)throw Error('Changed feature was not cached after activation'); await sleep(100); } } const duration = Date.now() - t0; proxy.stopTracking(); const stats = proxy.getStats(); assertChangedAssets(stats.requests, updateKind==='feature'?'/presenter.js':updateKind==='css'?'/theme-glass.css':'/app.js'); await browser.close(); return { updateKind, readyDuration, readyWireBytes, requests: stats.requests, duration, requestCount: stats.requestCount, wireBytes: stats.totalWireBytes, wireBytesByType: stats.bytesByType, }; } /** * Main execution */ async function main() { const options = parseArgs(); console.log('=== YTPlayer Performance Baseline Harness ==='); console.log(`Runs per scenario: ${options.runs}`); console.log(`Browser target: ${options.browser}`); console.log(`Network profile: ${options.profile}`); console.log(`Scenario target: ${options.scenario}`); const browsers = []; if (options.browser === 'all' || options.browser === 'chromium') browsers.push(pw.chromium); if (options.browser === 'all' || options.browser === 'webkit') browsers.push(pw.webkit); const profiles = []; if (options.profile === 'all' || options.profile === 'lte') profiles.push('lte'); if (options.profile === 'all' || options.profile === 'lossy') profiles.push('lossy'); const tmpRoot = path.join(REPO_ROOT, 'perf', '.tmp', `bench-${Date.now()}`); mkdirSync(tmpRoot, { recursive: true }); // Setup scratch trees for N -> N+1 update scenarios const treeBase = path.join(tmpRoot, 'tree-base'); const treeUpdateJs = path.join(tmpRoot, 'tree-update-js'); const treeUpdateCss = path.join(tmpRoot, 'tree-update-css'); const treeUpdateFeature = path.join(tmpRoot, 'tree-update-feature'); let frontendSource=path.join(REPO_ROOT,'frontend'); if(options.frontendCommit) { const archive=path.join(tmpRoot,'frontend.tar'); writeFileSync(archive,execFileSync('git',['archive',options.frontendCommit,'frontend'],{cwd:REPO_ROOT,maxBuffer:20*1024*1024})); execFileSync('tar',['-xf',archive,'-C',tmpRoot]); frontendSource=path.join(tmpRoot,'frontend'); } for(const tree of [treeBase,treeUpdateJs,treeUpdateCss,treeUpdateFeature])cpSync(frontendSource,tree,{recursive:true}); // Append 1 comment line to app.js in treeUpdateJs appendFileSync(path.join(treeUpdateJs, 'app.js'), '\n// baseline bench: 1-line app.js comment\n'); // Append 1 rule to theme-glass.css in treeUpdateCss appendFileSync(path.join(treeUpdateCss, 'theme-glass.css'), '\n/* baseline bench: 1-line css rule */\n.baseline-marker { opacity: 0.99; }\n'); if(existsSync(path.join(treeUpdateFeature,'presenter.js')))appendFileSync(path.join(treeUpdateFeature,'presenter.js'),'\n// baseline bench: 1-line presenter chunk comment\n'); const trees = { treeBase, treeUpdateJs, treeUpdateCss, treeUpdateFeature }; const serverScratch = path.join(tmpRoot, 'server-run'); const server = new ServerManager(serverScratch, treeBase); await server.start(); console.log(`Bun server running on ephemeral port ${server.port}`); const results = { date: new Date().toISOString(), frontendCommit: options.frontendCommit, frontendBuildTag: (await (await fetch('http://127.0.0.1:'+server.port+'/api/version')).json()).buildTag, commit: spawnSync('git', ['rev-parse', 'HEAD'], { cwd: REPO_ROOT }).stdout.toString().trim(), runsConfigured: options.runs, browsers: {}, }; try { for (const bType of browsers) { const bName = bType.name(); results.browsers[bName] = {}; for (const prof of profiles) { console.log(`\n>>> Running Benchmark: Browser=[${bName}] Profile=[${prof}]`); results.browsers[bName][prof] = {}; const proxyProfile = bName === 'chromium' ? 'unthrottled' : prof; const proxy = createThrottleProxy({ targetPort: server.port, profile: proxyProfile }); proxy.port = await proxy.listen(); console.log(`Proxy listening on port ${proxy.port} (mode: ${proxyProfile})`); // 1. Cold Scenario if (options.scenario === 'all' || options.scenario === 'cold') { console.log(` [Scenario: Cold Load] running ${options.runs} iterations...`); const coldRuns = []; for (let r = 0; r < options.runs; r++) { process.stdout.write(` run ${r + 1}/${options.runs}... `); const res = await runColdScenario({ browserType: bType, server, proxy, profile: prof }); coldRuns.push(res); console.log(`done (${res.wireBytes} bytes, ${res.requestCount} reqs, boot=${res.bootDone}ms)`); } results.browsers[bName][prof].cold = { requestCount: summarizeList(coldRuns.map((r) => r.requestCount)), wireBytes: summarizeList(coldRuns.map((r) => r.wireBytes)), fcp: summarizeList(coldRuns.map((r) => r.fcp)), lcp: summarizeList(coldRuns.map((r) => r.lcp)), domContentLoaded: summarizeList(coldRuns.map((r) => r.domContentLoaded)), bootDone: summarizeList(coldRuns.map((r) => r.bootDone)), longTaskTotal: summarizeList(coldRuns.map((r) => r.longTaskTotal)), longTasks: coldRuns.map(r=>r.longTasks), swInstallDuration: summarizeList(coldRuns.map((r) => r.swInstallDuration)), swInstallBytes: summarizeList(coldRuns.map((r) => r.swInstallBytes)), storageEstimate: coldRuns[0]?.storageEstimate || null, mediaReady: summarizeList(coldRuns.map((r) => r.mediaReady)), autoplayBlocked: coldRuns.some((r) => r.autoplayBlocked), wireBytesByType: coldRuns[0]?.wireBytesByType || {}, requests: coldRuns[0]?.requests || [], initialJsCss: summarizeList(coldRuns.map(r=>r.initialJsCss)), initialAssetRequests: coldRuns[0]?.initialAssetRequests || [], }; } // 2. Warm Scenario if (options.scenario === 'all' || options.scenario === 'warm') { console.log(` [Scenario: Warm Load] running ${options.runs} iterations...`); const warmRuns = []; for (let r = 0; r < options.runs; r++) { process.stdout.write(` run ${r + 1}/${options.runs}... `); const res = await runWarmScenario({ browserType: bType, server, proxy, profile: prof }); warmRuns.push(res); console.log(`done (${res.wireBytes} bytes, ${res.requestCount} reqs, boot=${res.bootDone}ms)`); } results.browsers[bName][prof].warm = { requestCount: summarizeList(warmRuns.map((r) => r.requestCount)), wireBytes: summarizeList(warmRuns.map((r) => r.wireBytes)), fcp: summarizeList(warmRuns.map((r) => r.fcp)), lcp: summarizeList(warmRuns.map((r) => r.lcp)), domContentLoaded: summarizeList(warmRuns.map((r) => r.domContentLoaded)), bootDone: summarizeList(warmRuns.map((r) => r.bootDone)), longTaskTotal: summarizeList(warmRuns.map((r) => r.longTaskTotal)), }; } // 3. Offline Scenario if (options.scenario === 'all' || options.scenario === 'offline') { console.log(` [Scenario: Offline Reload] running ${options.runs} iterations...`); const offRuns = []; for (let r = 0; r < options.runs; r++) { process.stdout.write(` run ${r + 1}/${options.runs}... `); const res = await runOfflineScenario({ browserType: bType, server, proxy, profile: prof }); offRuns.push(res); console.log(`done (reload=${res.reloadDuration}ms, boot=${res.bootDone}ms)`); } results.browsers[bName][prof].offline = { requestCount: summarizeList(offRuns.map((r) => r.requestCount)), wireBytes: summarizeList(offRuns.map((r) => r.wireBytes)), fcp: summarizeList(offRuns.map((r) => r.fcp)), lcp: summarizeList(offRuns.map((r) => r.lcp)), domContentLoaded: summarizeList(offRuns.map((r) => r.domContentLoaded)), bootDone: summarizeList(offRuns.map((r) => r.bootDone)), reloadDuration: summarizeList(offRuns.map((r) => r.reloadDuration)), }; } // 4. Update N -> N+1 (JS) if (options.scenario === 'all' || options.scenario === 'update-js') { console.log(` [Scenario: Update N->N+1 (1-line app.js)] running ${options.runs} iterations...`); const updJsRuns = []; for (let r = 0; r < options.runs; r++) { process.stdout.write(` run ${r + 1}/${options.runs}... `); const res = await runUpdateScenario({ browserType: bType, server, proxy, profile: prof, updateKind: 'js', trees, }); updJsRuns.push(res); console.log(`done (${res.wireBytes} bytes, ${res.requestCount} reqs, dur=${res.duration}ms)`); } results.browsers[bName][prof].updateJs = { requestCount: summarizeList(updJsRuns.map((r) => r.requestCount)), wireBytes: summarizeList(updJsRuns.map((r) => r.wireBytes)), duration: summarizeList(updJsRuns.map((r) => r.duration)), wireBytesByType: updJsRuns[0]?.wireBytesByType || {}, requests: updJsRuns[0]?.requests || [], }; } // 5. Update N -> N+1 (CSS) if (options.scenario === 'all' || options.scenario === 'update-css') { console.log(` [Scenario: Update N->N+1 (1-line theme-glass.css)] running ${options.runs} iterations...`); const updCssRuns = []; for (let r = 0; r < options.runs; r++) { process.stdout.write(` run ${r + 1}/${options.runs}... `); const res = await runUpdateScenario({ browserType: bType, server, proxy, profile: prof, updateKind: 'css', trees, }); updCssRuns.push(res); console.log(`done (${res.wireBytes} bytes, ${res.requestCount} reqs, dur=${res.duration}ms)`); } results.browsers[bName][prof].updateCss = { readyDuration: summarizeList(updCssRuns.map(r=>r.readyDuration)), readyWireBytes: summarizeList(updCssRuns.map(r=>r.readyWireBytes)), requestCount: summarizeList(updCssRuns.map((r) => r.requestCount)), wireBytes: summarizeList(updCssRuns.map((r) => r.wireBytes)), duration: summarizeList(updCssRuns.map((r) => r.duration)), wireBytesByType: updCssRuns[0]?.wireBytesByType || {}, requests: updCssRuns[0]?.requests || [], }; } if ((options.scenario === 'all' || options.scenario === 'update-feature') && existsSync(path.join(treeBase,'presenter.js'))) { console.log(` [Scenario: Update split-out presenter, including idle cache fill] running ${options.runs} iterations...`); const runs=[]; for(let r=0;rr.requestCount)),wireBytes:summarizeList(runs.map(r=>r.wireBytes)),duration:summarizeList(runs.map(r=>r.duration)),wireBytesByType:runs[0]?.wireBytesByType||{},requests:runs[0]?.requests||[]}; } await proxy.close(); } } } finally { await server.stop(); rmSync(tmpRoot, { recursive: true, force: true }); } // Write results JSON const now = new Date(); const dateStr = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`; const outPath = options.out || path.join(REPO_ROOT, 'perf', 'results', `baseline-${dateStr}.json`); mkdirSync(path.dirname(outPath), { recursive: true }); writeFileSync(outPath, JSON.stringify(results, null, 2)); console.log(`\nBaseline results saved to: ${outPath}`); // Print Summary Table printSummaryTable(results); // Compare mode if requested if (options.compare) { compareResults(options.compare, results); } } function printSummaryTable(results) { console.log('\n============================= HEADLINE BASELINE SUMMARY ============================='); for (const [browser, profiles] of Object.entries(results.browsers)) { for (const [profile, scenarios] of Object.entries(profiles)) { console.log(`\n[Browser: ${browser} | Profile: ${profile}]`); console.log('| Scenario | Req Count | Wire Bytes | Boot / Ready | SW Dur / Reload |'); console.log('|---------------------------|-----------|------------|--------------|-----------------|'); if (scenarios.cold) { const c = scenarios.cold; console.log( `| Cold First Load | ${String(c.requestCount.median).padEnd(9)} | ${formatBytes(c.wireBytes.median).padEnd(10)} | ${String(c.bootDone.median + 'ms').padEnd(12)} | ${String(c.swInstallDuration.median + 'ms').padEnd(15)} |` ); } if (scenarios.warm) { const w = scenarios.warm; console.log( `| Warm Reload (under SW) | ${String(w.requestCount.median).padEnd(9)} | ${formatBytes(w.wireBytes.median).padEnd(10)} | ${String(w.bootDone.median + 'ms').padEnd(12)} | ${'-'.padEnd(15)} |` ); } if (scenarios.offline) { const o = scenarios.offline; console.log( `| Offline Reload (under SW) | ${String(o.requestCount.median).padEnd(9)} | ${formatBytes(o.wireBytes.median).padEnd(10)} | ${String(o.bootDone.median + 'ms').padEnd(12)} | ${String(o.reloadDuration.median + 'ms').padEnd(15)} |` ); } if (scenarios.updateJs) { const u = scenarios.updateJs; console.log( `| Update N->N+1 (1-line JS) | ${String(u.requestCount.median).padEnd(9)} | ${formatBytes(u.wireBytes.median).padEnd(10)} | ${'-'.padEnd(12)} | ${String(u.duration.median + 'ms').padEnd(15)} |` ); } if (scenarios.updateCss) { const u = scenarios.updateCss; console.log( `| Update N->N+1 (1-line CSS)| ${String(u.requestCount.median).padEnd(9)} | ${formatBytes(u.wireBytes.median).padEnd(10)} | ${'-'.padEnd(12)} | ${String(u.duration.median + 'ms').padEnd(15)} |` ); } } } console.log('\n====================================================================================='); } function formatBytes(bytes) { if (bytes === null || bytes === undefined) return 'N/A'; if (bytes >= 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(2) + ' MB'; if (bytes >= 1024) return (bytes / 1024).toFixed(1) + ' KB'; return bytes + ' B'; } function compareResults(baselinePath, currentResults) { console.log(`\n================ Phase 5 Comparison against: ${baselinePath} ================`); if (!existsSync(baselinePath)) { console.error(`Error: Baseline file not found: ${baselinePath}`); return; } const base = JSON.parse(readFileSync(baselinePath, 'utf8')); for (const [browser, profiles] of Object.entries(currentResults.browsers)) { for (const [profile, scenarios] of Object.entries(profiles)) { console.log(`\n[Comparison: ${browser} / ${profile}]`); console.log('| Metric | Baseline | Current | Delta (%) |'); console.log('|------------------------------------|-------------|-------------|--------------|'); const baseScen = base.browsers?.[browser]?.[profile]; if (!baseScen) { console.log('| (No matching baseline data) | | | |'); continue; } const rows = [ ['Cold: Wire Bytes', baseScen.cold?.wireBytes?.median, scenarios.cold?.wireBytes?.median, true], ['Cold: Boot Done (ms)', baseScen.cold?.bootDone?.median, scenarios.cold?.bootDone?.median, false], ['Cold: Requests', baseScen.cold?.requestCount?.median, scenarios.cold?.requestCount?.median, false], ['Warm: Boot Done (ms)', baseScen.warm?.bootDone?.median, scenarios.warm?.bootDone?.median, false], ['Update (1-line JS): Wire Bytes', baseScen.updateJs?.wireBytes?.median, scenarios.updateJs?.wireBytes?.median, true], ['Update (1-line JS): Time (ms)', baseScen.updateJs?.duration?.median, scenarios.updateJs?.duration?.median, false], ['Update (1-line CSS): Wire Bytes', baseScen.updateCss?.wireBytes?.median, scenarios.updateCss?.wireBytes?.median, true], ['Update (1-line CSS): Time (ms)', baseScen.updateCss?.duration?.median, scenarios.updateCss?.duration?.median, false], ]; for (const [name, bVal, cVal, isBytes] of rows) { if (bVal === undefined || cVal === undefined) continue; const bStr = isBytes ? formatBytes(bVal) : bVal !== null ? `${bVal}` : 'N/A'; const cStr = isBytes ? formatBytes(cVal) : cVal !== null ? `${cVal}` : 'N/A'; let deltaStr = '-'; if (typeof bVal === 'number' && typeof cVal === 'number' && bVal > 0) { const pct = (((cVal - bVal) / bVal) * 100).toFixed(1); const sign = cVal >= bVal ? '+' : ''; deltaStr = `${sign}${pct}%`; } console.log(`| ${name.padEnd(34)} | ${bStr.padEnd(11)} | ${cStr.padEnd(11)} | ${deltaStr.padEnd(12)} |`); } } } console.log('=============================================================================='); } main().catch((err) => { console.error('Benchmark fatal error:', err); process.exit(1); });