955 lines
36 KiB
JavaScript
Executable File
955 lines
36 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
|
|
/**
|
|
* perf/baseline.mjs
|
|
*
|
|
* Performance baseline harness for ytplayer PWA.
|
|
* Drives the real Bun server and runs Playwright (Chromium with CDP throttling + CPU 4x;
|
|
* WebKit with a local throttling & stalling proxy).
|
|
*
|
|
* Scenarios:
|
|
* - cold: fresh context, network download, SW install, boot, media playback check
|
|
* - warm: re-navigate with active SW and primed HTTP cache
|
|
* - offline: context.setOffline(true), reload under SW cache
|
|
* - update-js: update N->N+1 (1-line comment appended to app.js) -> wire bytes + time until new SW ready
|
|
* - update-css: update N->N+1 (1-line rule appended to theme-glass.css) -> wire bytes + time until new SW ready
|
|
*
|
|
* Usage:
|
|
* node perf/baseline.mjs [options]
|
|
* Options:
|
|
* --runs <number> Runs per scenario for median calculation (default: 5)
|
|
* --browser <chr|wk|all> Browser(s) to test (default: all)
|
|
* --profile <lte|lossy|all> Network profile (default: lte)
|
|
* --scenario <name|all> Specific scenario or all (default: all)
|
|
* --out <path> Output JSON path
|
|
* --compare <json-path> Compare results against a previous baseline JSON
|
|
*/
|
|
|
|
import { spawn, spawnSync } from 'node:child_process';
|
|
import { existsSync, mkdirSync, rmSync, cpSync, appendFileSync, statSync, writeFileSync, readFileSync, symlinkSync, unlinkSync } from 'node:fs';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { createThrottleProxy } from './proxy.mjs';
|
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = path.dirname(__filename);
|
|
const REPO_ROOT = path.resolve(__dirname, '..');
|
|
|
|
// Dynamically resolve playwright from node_modules
|
|
let pw;
|
|
try {
|
|
pw = await import('playwright');
|
|
} catch {
|
|
pw = await import(path.join(REPO_ROOT, 'node_modules', 'playwright', 'index.mjs'));
|
|
}
|
|
|
|
import { createRequire } from 'node:module';
|
|
const req = createRequire(import.meta.url);
|
|
const { createClient } = req(path.join(REPO_ROOT, 'server', 'node_modules', '@libsql/client'));
|
|
|
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
function parseArgs() {
|
|
const args = process.argv.slice(2);
|
|
const options = {
|
|
runs: 5,
|
|
browser: 'all', // 'chromium', 'webkit', or 'all'
|
|
profile: 'lte', // 'lte', 'lossy', or 'all'
|
|
scenario: 'all', // 'cold', 'warm', 'offline', 'update-js', 'update-css', or 'all'
|
|
out: null,
|
|
compare: null,
|
|
};
|
|
|
|
for (let i = 0; i < args.length; i++) {
|
|
const a = args[i];
|
|
if (a === '--runs' && i + 1 < args.length) options.runs = parseInt(args[++i], 10);
|
|
else if (a === '--browser' && i + 1 < args.length) options.browser = args[++i];
|
|
else if (a === '--profile' && i + 1 < args.length) options.profile = args[++i];
|
|
else if (a === '--scenario' && i + 1 < args.length) options.scenario = args[++i];
|
|
else if (a === '--out' && i + 1 < args.length) options.out = args[++i];
|
|
else if (a === '--compare' && i + 1 < args.length) options.compare = args[++i];
|
|
}
|
|
|
|
return options;
|
|
}
|
|
|
|
function median(values) {
|
|
const nums = values.filter((v) => typeof v === 'number' && !Number.isNaN(v)).sort((a, b) => a - b);
|
|
if (!nums.length) return null;
|
|
const mid = Math.floor(nums.length / 2);
|
|
return nums.length % 2 !== 0 ? nums[mid] : Math.round(((nums[mid - 1] + nums[mid]) / 2) * 10) / 10;
|
|
}
|
|
|
|
function summarizeList(values) {
|
|
const valid = values.filter((v) => v !== null && v !== undefined && !Number.isNaN(v));
|
|
if (!valid.length) return { median: null, min: null, max: null, values };
|
|
return {
|
|
median: median(valid),
|
|
min: Math.min(...valid),
|
|
max: Math.max(...valid),
|
|
values,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Setup deterministic local media fixture
|
|
*/
|
|
function prepareMediaFiles(mediaDir) {
|
|
mkdirSync(mediaDir, { recursive: true });
|
|
const mp4 = path.join(mediaDir, 'benchmedia01.1.mp4');
|
|
const m4a = path.join(mediaDir, 'benchmedia01.1.m4a');
|
|
|
|
if (!existsSync(mp4)) {
|
|
spawnSync('ffmpeg', [
|
|
'-y', '-f', 'lavfi', '-i', 'testsrc=size=640x360:rate=30:duration=5',
|
|
'-f', 'lavfi', '-i', 'sine=frequency=440:duration=5',
|
|
'-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p',
|
|
'-c:a', 'aac', '-movflags', '+faststart', '-shortest', mp4,
|
|
]);
|
|
}
|
|
|
|
if (!existsSync(m4a)) {
|
|
spawnSync('ffmpeg', [
|
|
'-y', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=5',
|
|
'-c:a', 'aac', '-shortest', m4a,
|
|
]);
|
|
}
|
|
|
|
return { mp4, m4a };
|
|
}
|
|
|
|
async function seedDatabase(dbPath, mediaFiles) {
|
|
const db = createClient({ url: 'file:' + dbPath });
|
|
const mp4Size = statSync(mediaFiles.mp4).size;
|
|
const m4aSize = statSync(mediaFiles.m4a).size;
|
|
|
|
await db.execute({
|
|
sql: `INSERT OR REPLACE INTO media_cache (
|
|
video_id, status, gen, size, height, vcodec, acodec, duration, meta, last_access
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
args: [
|
|
'benchmedia01',
|
|
'ready',
|
|
1,
|
|
mp4Size + m4aSize,
|
|
360,
|
|
'h264',
|
|
'aac',
|
|
5.0,
|
|
JSON.stringify({ title: 'Baseline Deterministic Clip', channel: 'Perf Benchmark' }),
|
|
Date.now(),
|
|
],
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Bun Server Manager
|
|
*/
|
|
class ServerManager {
|
|
constructor(scratchDir, publicDir) {
|
|
this.scratchDir = scratchDir;
|
|
this.publicDir = publicDir;
|
|
this.dataDir = path.join(scratchDir, 'data');
|
|
this.mediaDir = path.join(this.dataDir, 'media');
|
|
this.dbPath = path.join(this.dataDir, 'ytplayer.db');
|
|
this.serverProcess = null;
|
|
this.port = 0;
|
|
}
|
|
|
|
async start() {
|
|
mkdirSync(this.dataDir, { recursive: true });
|
|
const fixtureDir = path.join(REPO_ROOT, 'perf', 'fixtures', 'media');
|
|
const mediaFiles = prepareMediaFiles(fixtureDir);
|
|
|
|
// Create a dedicated server working dir with public symlink
|
|
const srvCwd = path.join(this.scratchDir, 'srv');
|
|
mkdirSync(srvCwd, { recursive: true });
|
|
const publicLink = path.join(srvCwd, 'public');
|
|
if (existsSync(publicLink)) unlinkSync(publicLink);
|
|
symlinkSync(this.publicDir, publicLink);
|
|
|
|
// Pick random available ephemeral port once and reuse across restarts
|
|
this.port = this.port || (4300 + Math.floor(Math.random() * 2000));
|
|
|
|
const env = {
|
|
...process.env,
|
|
PORT: String(this.port),
|
|
DATA_DIR: this.dataDir,
|
|
MEDIA_DIR: this.mediaDir,
|
|
DB_PATH: this.dbPath,
|
|
};
|
|
|
|
const serverScript = path.join(REPO_ROOT, 'server', 'server.js');
|
|
this.serverProcess = spawn('bun', [serverScript], {
|
|
cwd: srvCwd,
|
|
env,
|
|
stdio: 'ignore',
|
|
});
|
|
|
|
// Wait for server readiness
|
|
let ready = false;
|
|
for (let i = 0; i < 50; i++) {
|
|
try {
|
|
const res = await fetch(`http://127.0.0.1:${this.port}/api/version`);
|
|
if (res.ok) {
|
|
ready = true;
|
|
break;
|
|
}
|
|
} catch {}
|
|
await sleep(100);
|
|
}
|
|
|
|
if (!ready) {
|
|
throw new Error(`Server failed to start on port ${this.port}`);
|
|
}
|
|
|
|
// Copy media files to mediaDir and seed media table AFTER server startup sweep
|
|
mkdirSync(this.mediaDir, { recursive: true });
|
|
const targetMp4 = path.join(this.mediaDir, 'benchmedia01.1.mp4');
|
|
const targetM4a = path.join(this.mediaDir, 'benchmedia01.1.m4a');
|
|
cpSync(mediaFiles.mp4, targetMp4);
|
|
cpSync(mediaFiles.m4a, targetM4a);
|
|
await seedDatabase(this.dbPath, { mp4: targetMp4, m4a: targetM4a });
|
|
}
|
|
|
|
async stop() {
|
|
if (this.serverProcess) {
|
|
const proc = this.serverProcess;
|
|
this.serverProcess = null;
|
|
await new Promise((resolve) => {
|
|
if (proc.exitCode !== null) return resolve();
|
|
proc.once('exit', () => resolve());
|
|
proc.kill('SIGKILL');
|
|
setTimeout(resolve, 2000);
|
|
});
|
|
await sleep(200);
|
|
}
|
|
}
|
|
|
|
async restartWithPublic(newPublicDir) {
|
|
await this.stop();
|
|
this.publicDir = newPublicDir;
|
|
await this.start();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Configure page for metrics collection
|
|
*/
|
|
async function setupPageMetrics(page) {
|
|
await page.addInitScript(() => {
|
|
window.__bootDone = null;
|
|
window.__fcp = null;
|
|
window.__lcp = null;
|
|
window.__longTasksTotal = 0;
|
|
|
|
// Observe boot completion landmark (#cards view set by renderList())
|
|
const markBoot = () => {
|
|
const cards = document.getElementById('cards');
|
|
if (cards && cards.dataset && cards.dataset.view) {
|
|
if (!window.__bootDone) window.__bootDone = performance.now();
|
|
return true;
|
|
}
|
|
return false;
|
|
};
|
|
const t = setInterval(() => {
|
|
if (markBoot()) clearInterval(t);
|
|
}, 5);
|
|
|
|
// Observe Long Tasks API (supported on Chromium)
|
|
try {
|
|
new PerformanceObserver((list) => {
|
|
for (const entry of list.getEntries()) {
|
|
window.__longTasksTotal += entry.duration;
|
|
}
|
|
}).observe({ type: 'longtask', buffered: true });
|
|
} catch {}
|
|
|
|
// Observe LCP
|
|
try {
|
|
new PerformanceObserver((list) => {
|
|
const entries = list.getEntries();
|
|
if (entries.length > 0) {
|
|
window.__lcp = entries[entries.length - 1].startTime;
|
|
}
|
|
}).observe({ type: 'largest-contentful-paint', buffered: true });
|
|
} catch {}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Run Scenario: Cold Load
|
|
*/
|
|
async function runColdScenario({ browserType, server, proxy, profile }) {
|
|
const isWebKit = browserType.name() === 'webkit';
|
|
const launchArgs = isWebKit ? [] : ['--autoplay-policy=no-user-gesture-required'];
|
|
const browser = await browserType.launch({
|
|
headless: true,
|
|
args: launchArgs,
|
|
});
|
|
|
|
const contextOptions = { proxy: { server: `http://127.0.0.1:${proxy.port}` } };
|
|
const context = await browser.newContext(contextOptions);
|
|
const page = await context.newPage();
|
|
|
|
// If Chromium, configure CDP Network conditions and 4x CPU throttle
|
|
if (!isWebKit) {
|
|
const cdp = await context.newCDPSession(page);
|
|
await cdp.send('Network.enable');
|
|
const isLossy = profile === 'lossy';
|
|
await cdp.send('Network.emulateNetworkConditions', {
|
|
offline: false,
|
|
latency: isLossy ? 250 : 150,
|
|
downloadThroughput: isLossy ? (1.0 * 1000 * 1000) / 8 : (1.6 * 1000 * 1000) / 8,
|
|
uploadThroughput: (750 * 1000) / 8,
|
|
connectionType: 'cellular4g',
|
|
});
|
|
await cdp.send('Emulation.setCPUThrottlingRate', { rate: 4 });
|
|
}
|
|
|
|
await setupPageMetrics(page);
|
|
|
|
proxy.resetStats();
|
|
proxy.startTracking();
|
|
|
|
const swInstallStart = Date.now();
|
|
const navUrl = `http://127.0.0.1:${server.port}/?v=benchmedia01`;
|
|
|
|
await page.goto(navUrl);
|
|
|
|
// Wait for boot landmark
|
|
await page.waitForFunction(() => window.__bootDone !== null, { timeout: 15000 });
|
|
|
|
// Check deterministic media playing-ready
|
|
let mediaReadyMs = null;
|
|
let autoplayBlocked = false;
|
|
try {
|
|
const mediaCheck = await page.waitForFunction(() => {
|
|
const v = document.querySelector('video') || document.querySelector('audio');
|
|
if (!v) return false;
|
|
return v.readyState >= 3 || (!v.paused && v.currentTime > 0);
|
|
}, { timeout: 8000 });
|
|
if (mediaCheck) {
|
|
const state = await page.evaluate(() => {
|
|
const v = document.querySelector('video') || document.querySelector('audio');
|
|
return { paused: v.paused, readyState: v.readyState };
|
|
});
|
|
autoplayBlocked = state.paused;
|
|
mediaReadyMs = await page.evaluate(() => performance.now());
|
|
}
|
|
} catch {
|
|
autoplayBlocked = true;
|
|
}
|
|
|
|
// Wait for initial Service Worker activation
|
|
await page.evaluate(async () => {
|
|
const reg = await navigator.serviceWorker.ready;
|
|
if (reg.active && reg.active.state === 'activated') return;
|
|
await new Promise((resolve) => {
|
|
const w = reg.active || reg.installing;
|
|
if (w.state === 'activated') return resolve();
|
|
w.addEventListener('statechange', () => {
|
|
if (w.state === 'activated') resolve();
|
|
});
|
|
});
|
|
});
|
|
|
|
const swInstallDuration = Date.now() - swInstallStart;
|
|
proxy.stopTracking();
|
|
|
|
const stats = proxy.getStats();
|
|
|
|
const pageMetrics = await page.evaluate(() => {
|
|
const nav = performance.getEntriesByType('navigation')[0];
|
|
const paint = performance.getEntriesByType('paint');
|
|
const fcp = paint.find((p) => p.name === 'first-contentful-paint');
|
|
|
|
return {
|
|
fcp: fcp ? Math.round(fcp.startTime) : null,
|
|
lcp: window.__lcp ? Math.round(window.__lcp) : null,
|
|
domContentLoaded: nav ? Math.round(nav.domContentLoadedEventEnd - nav.startTime) : null,
|
|
bootDone: window.__bootDone ? Math.round(window.__bootDone) : null,
|
|
longTaskTotal: Math.round(window.__longTasksTotal || 0),
|
|
};
|
|
});
|
|
|
|
const storage = await page.evaluate(async () => {
|
|
if (navigator.storage && navigator.storage.estimate) {
|
|
const est = await navigator.storage.estimate();
|
|
return { usage: est.usage || 0, quota: est.quota || 0 };
|
|
}
|
|
return { usage: null, quota: null };
|
|
});
|
|
|
|
await browser.close();
|
|
|
|
return {
|
|
requestCount: stats.requestCount,
|
|
wireBytes: stats.totalWireBytes,
|
|
wireBytesByType: stats.bytesByType,
|
|
fcp: pageMetrics.fcp,
|
|
lcp: pageMetrics.lcp,
|
|
domContentLoaded: pageMetrics.domContentLoaded,
|
|
bootDone: pageMetrics.bootDone,
|
|
longTaskTotal: pageMetrics.longTaskTotal,
|
|
swInstallDuration,
|
|
swInstallBytes: stats.totalWireBytes,
|
|
storageEstimate: storage,
|
|
mediaReady: mediaReadyMs ? Math.round(mediaReadyMs) : null,
|
|
autoplayBlocked,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Run Scenario: Warm Load (re-navigation with active SW)
|
|
*/
|
|
async function runWarmScenario({ browserType, server, proxy, profile }) {
|
|
const isWebKit = browserType.name() === 'webkit';
|
|
const launchArgs = isWebKit ? [] : ['--autoplay-policy=no-user-gesture-required'];
|
|
const browser = await browserType.launch({
|
|
headless: true,
|
|
args: launchArgs,
|
|
});
|
|
|
|
const contextOptions = { proxy: { server: `http://127.0.0.1:${proxy.port}` } };
|
|
const context = await browser.newContext(contextOptions);
|
|
const page = await context.newPage();
|
|
|
|
if (!isWebKit) {
|
|
const cdp = await context.newCDPSession(page);
|
|
await cdp.send('Network.enable');
|
|
const isLossy = profile === 'lossy';
|
|
await cdp.send('Network.emulateNetworkConditions', {
|
|
offline: false,
|
|
latency: isLossy ? 250 : 150,
|
|
downloadThroughput: isLossy ? (1.0 * 1000 * 1000) / 8 : (1.6 * 1000 * 1000) / 8,
|
|
uploadThroughput: (750 * 1000) / 8,
|
|
connectionType: 'cellular4g',
|
|
});
|
|
await cdp.send('Emulation.setCPUThrottlingRate', { rate: 4 });
|
|
}
|
|
|
|
await setupPageMetrics(page);
|
|
|
|
const navUrl = `http://127.0.0.1:${server.port}/`;
|
|
|
|
// Prime context with cold load
|
|
await page.goto(navUrl);
|
|
await page.waitForFunction(() => window.__bootDone !== null, { timeout: 15000 });
|
|
await page.evaluate(async () => {
|
|
const reg = await navigator.serviceWorker.ready;
|
|
if (reg.active && reg.active.state === 'activated') return;
|
|
await new Promise((resolve) => {
|
|
const w = reg.active || reg.installing;
|
|
if (w.state === 'activated') return resolve();
|
|
w.addEventListener('statechange', () => {
|
|
if (w.state === 'activated') resolve();
|
|
});
|
|
});
|
|
});
|
|
|
|
// Now measure warm re-navigation
|
|
proxy.resetStats();
|
|
proxy.startTracking();
|
|
|
|
await page.goto(navUrl);
|
|
await page.waitForFunction(() => window.__bootDone !== null, { timeout: 15000 });
|
|
|
|
proxy.stopTracking();
|
|
const stats = proxy.getStats();
|
|
|
|
const pageMetrics = await page.evaluate(() => {
|
|
const nav = performance.getEntriesByType('navigation')[0];
|
|
const paint = performance.getEntriesByType('paint');
|
|
const fcp = paint.find((p) => p.name === 'first-contentful-paint');
|
|
|
|
return {
|
|
fcp: fcp ? Math.round(fcp.startTime) : null,
|
|
lcp: window.__lcp ? Math.round(window.__lcp) : null,
|
|
domContentLoaded: nav ? Math.round(nav.domContentLoadedEventEnd - nav.startTime) : null,
|
|
bootDone: window.__bootDone ? Math.round(window.__bootDone) : null,
|
|
longTaskTotal: Math.round(window.__longTasksTotal || 0),
|
|
};
|
|
});
|
|
|
|
await browser.close();
|
|
|
|
return {
|
|
requestCount: stats.requestCount,
|
|
wireBytes: stats.totalWireBytes,
|
|
wireBytesByType: stats.bytesByType,
|
|
fcp: pageMetrics.fcp,
|
|
lcp: pageMetrics.lcp,
|
|
domContentLoaded: pageMetrics.domContentLoaded,
|
|
bootDone: pageMetrics.bootDone,
|
|
longTaskTotal: pageMetrics.longTaskTotal,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Run Scenario: Offline Reload under SW
|
|
*/
|
|
async function runOfflineScenario({ browserType, server, proxy, profile }) {
|
|
const browser = await browserType.launch({ headless: true });
|
|
const isWebKit = browserType.name() === 'webkit';
|
|
const contextOptions = { proxy: { server: `http://127.0.0.1:${proxy.port}` } };
|
|
const context = await browser.newContext(contextOptions);
|
|
const page = await context.newPage();
|
|
|
|
await setupPageMetrics(page);
|
|
const navUrl = `http://127.0.0.1:${server.port}/`;
|
|
|
|
// Prime SW
|
|
await page.goto(navUrl);
|
|
await page.waitForFunction(() => window.__bootDone !== null, { timeout: 15000 });
|
|
await page.evaluate(async () => {
|
|
const reg = await navigator.serviceWorker.ready;
|
|
if (reg.active && reg.active.state === 'activated') return;
|
|
await new Promise((resolve) => {
|
|
const w = reg.active || reg.installing;
|
|
if (w.state === 'activated') return resolve();
|
|
w.addEventListener('statechange', () => {
|
|
if (w.state === 'activated') resolve();
|
|
});
|
|
});
|
|
});
|
|
|
|
// Switch to offline mode:
|
|
// On Chromium, context.setOffline(true) works.
|
|
// On Linux WebKit, Playwright context.setOffline throws an internal browser error,
|
|
// so proxy.setOffline(true) cuts upstream network to simulate offline truthfully.
|
|
if (!isWebKit) {
|
|
await context.setOffline(true);
|
|
}
|
|
proxy.setOffline(true);
|
|
|
|
proxy.resetStats();
|
|
proxy.startTracking();
|
|
|
|
const t0 = Date.now();
|
|
await page.reload();
|
|
await page.waitForFunction(() => window.__bootDone !== null, { timeout: 15000 });
|
|
const reloadDuration = Date.now() - t0;
|
|
|
|
proxy.stopTracking();
|
|
proxy.setOffline(false);
|
|
const stats = proxy.getStats();
|
|
|
|
const pageMetrics = await page.evaluate(() => {
|
|
const nav = performance.getEntriesByType('navigation')[0];
|
|
const paint = performance.getEntriesByType('paint');
|
|
const fcp = paint.find((p) => p.name === 'first-contentful-paint');
|
|
|
|
return {
|
|
fcp: fcp ? Math.round(fcp.startTime) : null,
|
|
lcp: window.__lcp ? Math.round(window.__lcp) : null,
|
|
domContentLoaded: nav ? Math.round(nav.domContentLoadedEventEnd - nav.startTime) : null,
|
|
bootDone: window.__bootDone ? Math.round(window.__bootDone) : null,
|
|
};
|
|
});
|
|
|
|
await browser.close();
|
|
|
|
return {
|
|
requestCount: stats.requestCount,
|
|
wireBytes: stats.totalWireBytes,
|
|
fcp: pageMetrics.fcp,
|
|
lcp: pageMetrics.lcp,
|
|
domContentLoaded: pageMetrics.domContentLoaded,
|
|
bootDone: pageMetrics.bootDone,
|
|
reloadDuration,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Run Scenario: Update N -> N+1
|
|
*/
|
|
async function runUpdateScenario({ browserType, server, proxy, profile, updateKind, trees }) {
|
|
const browser = await browserType.launch({ headless: true });
|
|
const isWebKit = browserType.name() === 'webkit';
|
|
const contextOptions = { proxy: { server: `http://127.0.0.1:${proxy.port}` } };
|
|
const context = await browser.newContext(contextOptions);
|
|
const page = await context.newPage();
|
|
|
|
if (!isWebKit) {
|
|
const cdp = await context.newCDPSession(page);
|
|
await cdp.send('Network.enable');
|
|
const isLossy = profile === 'lossy';
|
|
await cdp.send('Network.emulateNetworkConditions', {
|
|
offline: false,
|
|
latency: isLossy ? 250 : 150,
|
|
downloadThroughput: isLossy ? (1.0 * 1000 * 1000) / 8 : (1.6 * 1000 * 1000) / 8,
|
|
uploadThroughput: (750 * 1000) / 8,
|
|
connectionType: 'cellular4g',
|
|
});
|
|
}
|
|
|
|
// Start on Tree N (base)
|
|
await server.restartWithPublic(trees.treeBase);
|
|
|
|
const navUrl = `http://127.0.0.1:${server.port}/`;
|
|
|
|
await page.goto(navUrl);
|
|
|
|
// Wait for steady state: active worker present, no waiting worker
|
|
while (true) {
|
|
const ready = await page.evaluate(async () => {
|
|
const reg = await navigator.serviceWorker.getRegistration();
|
|
return reg?.active?.state === 'activated' && !reg?.installing && !reg?.waiting;
|
|
});
|
|
if (ready) break;
|
|
await sleep(50);
|
|
}
|
|
|
|
// Switch server to Tree N+1 (app.js or theme-glass.css modification)
|
|
const targetTree = updateKind === 'js' ? trees.treeUpdateJs : trees.treeUpdateCss;
|
|
await server.restartWithPublic(targetTree);
|
|
|
|
// Measure wire bytes and duration from reg.update() until new SW is waiting/installed
|
|
proxy.resetStats();
|
|
proxy.startTracking();
|
|
|
|
const t0 = Date.now();
|
|
await page.evaluate(() => navigator.serviceWorker.getRegistration().then((r) => r.update()));
|
|
|
|
// Wait until new worker reaches waiting state
|
|
while (true) {
|
|
const waiting = await page.evaluate(async () => {
|
|
const reg = await navigator.serviceWorker.getRegistration();
|
|
return reg?.waiting?.state === 'installed';
|
|
});
|
|
if (waiting) break;
|
|
await sleep(50);
|
|
}
|
|
|
|
const duration = Date.now() - t0;
|
|
proxy.stopTracking();
|
|
const stats = proxy.getStats();
|
|
|
|
await browser.close();
|
|
|
|
return {
|
|
updateKind,
|
|
duration,
|
|
requestCount: stats.requestCount,
|
|
wireBytes: stats.totalWireBytes,
|
|
wireBytesByType: stats.bytesByType,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Main execution
|
|
*/
|
|
async function main() {
|
|
const options = parseArgs();
|
|
console.log('=== YTPlayer Performance Baseline Harness ===');
|
|
console.log(`Runs per scenario: ${options.runs}`);
|
|
console.log(`Browser target: ${options.browser}`);
|
|
console.log(`Network profile: ${options.profile}`);
|
|
console.log(`Scenario target: ${options.scenario}`);
|
|
|
|
const browsers = [];
|
|
if (options.browser === 'all' || options.browser === 'chromium') browsers.push(pw.chromium);
|
|
if (options.browser === 'all' || options.browser === 'webkit') browsers.push(pw.webkit);
|
|
|
|
const profiles = [];
|
|
if (options.profile === 'all' || options.profile === 'lte') profiles.push('lte');
|
|
if (options.profile === 'all' || options.profile === 'lossy') profiles.push('lossy');
|
|
|
|
const tmpRoot = path.join(REPO_ROOT, 'perf', '.tmp', `bench-${Date.now()}`);
|
|
mkdirSync(tmpRoot, { recursive: true });
|
|
|
|
// Setup scratch trees for N -> N+1 update scenarios
|
|
const treeBase = path.join(tmpRoot, 'tree-base');
|
|
const treeUpdateJs = path.join(tmpRoot, 'tree-update-js');
|
|
const treeUpdateCss = path.join(tmpRoot, 'tree-update-css');
|
|
|
|
cpSync(path.join(REPO_ROOT, 'frontend'), treeBase, { recursive: true });
|
|
cpSync(path.join(REPO_ROOT, 'frontend'), treeUpdateJs, { recursive: true });
|
|
cpSync(path.join(REPO_ROOT, 'frontend'), treeUpdateCss, { recursive: true });
|
|
|
|
// Append 1 comment line to app.js in treeUpdateJs
|
|
appendFileSync(path.join(treeUpdateJs, 'app.js'), '\n// baseline bench: 1-line app.js comment\n');
|
|
// Append 1 rule to theme-glass.css in treeUpdateCss
|
|
appendFileSync(path.join(treeUpdateCss, 'theme-glass.css'), '\n/* baseline bench: 1-line css rule */\n.baseline-marker { opacity: 0.99; }\n');
|
|
|
|
const trees = { treeBase, treeUpdateJs, treeUpdateCss };
|
|
|
|
const serverScratch = path.join(tmpRoot, 'server-run');
|
|
const server = new ServerManager(serverScratch, treeBase);
|
|
await server.start();
|
|
console.log(`Bun server running on ephemeral port ${server.port}`);
|
|
|
|
const results = {
|
|
date: new Date().toISOString(),
|
|
commit: spawnSync('git', ['rev-parse', 'HEAD'], { cwd: REPO_ROOT }).stdout.toString().trim(),
|
|
runsConfigured: options.runs,
|
|
browsers: {},
|
|
};
|
|
|
|
try {
|
|
for (const bType of browsers) {
|
|
const bName = bType.name();
|
|
results.browsers[bName] = {};
|
|
|
|
for (const prof of profiles) {
|
|
console.log(`\n>>> Running Benchmark: Browser=[${bName}] Profile=[${prof}]`);
|
|
results.browsers[bName][prof] = {};
|
|
|
|
const proxyProfile = bName === 'chromium' ? 'unthrottled' : prof;
|
|
const proxy = createThrottleProxy({ targetPort: server.port, profile: proxyProfile });
|
|
proxy.port = await proxy.listen();
|
|
console.log(`Proxy listening on port ${proxy.port} (mode: ${proxyProfile})`);
|
|
|
|
// 1. Cold Scenario
|
|
if (options.scenario === 'all' || options.scenario === 'cold') {
|
|
console.log(` [Scenario: Cold Load] running ${options.runs} iterations...`);
|
|
const coldRuns = [];
|
|
for (let r = 0; r < options.runs; r++) {
|
|
process.stdout.write(` run ${r + 1}/${options.runs}... `);
|
|
const res = await runColdScenario({ browserType: bType, server, proxy, profile: prof });
|
|
coldRuns.push(res);
|
|
console.log(`done (${res.wireBytes} bytes, ${res.requestCount} reqs, boot=${res.bootDone}ms)`);
|
|
}
|
|
|
|
results.browsers[bName][prof].cold = {
|
|
requestCount: summarizeList(coldRuns.map((r) => r.requestCount)),
|
|
wireBytes: summarizeList(coldRuns.map((r) => r.wireBytes)),
|
|
fcp: summarizeList(coldRuns.map((r) => r.fcp)),
|
|
lcp: summarizeList(coldRuns.map((r) => r.lcp)),
|
|
domContentLoaded: summarizeList(coldRuns.map((r) => r.domContentLoaded)),
|
|
bootDone: summarizeList(coldRuns.map((r) => r.bootDone)),
|
|
longTaskTotal: summarizeList(coldRuns.map((r) => r.longTaskTotal)),
|
|
swInstallDuration: summarizeList(coldRuns.map((r) => r.swInstallDuration)),
|
|
swInstallBytes: summarizeList(coldRuns.map((r) => r.swInstallBytes)),
|
|
storageEstimate: coldRuns[0]?.storageEstimate || null,
|
|
mediaReady: summarizeList(coldRuns.map((r) => r.mediaReady)),
|
|
autoplayBlocked: coldRuns.some((r) => r.autoplayBlocked),
|
|
wireBytesByType: coldRuns[0]?.wireBytesByType || {},
|
|
};
|
|
}
|
|
|
|
// 2. Warm Scenario
|
|
if (options.scenario === 'all' || options.scenario === 'warm') {
|
|
console.log(` [Scenario: Warm Load] running ${options.runs} iterations...`);
|
|
const warmRuns = [];
|
|
for (let r = 0; r < options.runs; r++) {
|
|
process.stdout.write(` run ${r + 1}/${options.runs}... `);
|
|
const res = await runWarmScenario({ browserType: bType, server, proxy, profile: prof });
|
|
warmRuns.push(res);
|
|
console.log(`done (${res.wireBytes} bytes, ${res.requestCount} reqs, boot=${res.bootDone}ms)`);
|
|
}
|
|
|
|
results.browsers[bName][prof].warm = {
|
|
requestCount: summarizeList(warmRuns.map((r) => r.requestCount)),
|
|
wireBytes: summarizeList(warmRuns.map((r) => r.wireBytes)),
|
|
fcp: summarizeList(warmRuns.map((r) => r.fcp)),
|
|
lcp: summarizeList(warmRuns.map((r) => r.lcp)),
|
|
domContentLoaded: summarizeList(warmRuns.map((r) => r.domContentLoaded)),
|
|
bootDone: summarizeList(warmRuns.map((r) => r.bootDone)),
|
|
longTaskTotal: summarizeList(warmRuns.map((r) => r.longTaskTotal)),
|
|
};
|
|
}
|
|
|
|
// 3. Offline Scenario
|
|
if (options.scenario === 'all' || options.scenario === 'offline') {
|
|
console.log(` [Scenario: Offline Reload] running ${options.runs} iterations...`);
|
|
const offRuns = [];
|
|
for (let r = 0; r < options.runs; r++) {
|
|
process.stdout.write(` run ${r + 1}/${options.runs}... `);
|
|
const res = await runOfflineScenario({ browserType: bType, server, proxy, profile: prof });
|
|
offRuns.push(res);
|
|
console.log(`done (reload=${res.reloadDuration}ms, boot=${res.bootDone}ms)`);
|
|
}
|
|
|
|
results.browsers[bName][prof].offline = {
|
|
requestCount: summarizeList(offRuns.map((r) => r.requestCount)),
|
|
wireBytes: summarizeList(offRuns.map((r) => r.wireBytes)),
|
|
fcp: summarizeList(offRuns.map((r) => r.fcp)),
|
|
lcp: summarizeList(offRuns.map((r) => r.lcp)),
|
|
domContentLoaded: summarizeList(offRuns.map((r) => r.domContentLoaded)),
|
|
bootDone: summarizeList(offRuns.map((r) => r.bootDone)),
|
|
reloadDuration: summarizeList(offRuns.map((r) => r.reloadDuration)),
|
|
};
|
|
}
|
|
|
|
// 4. Update N -> N+1 (JS)
|
|
if (options.scenario === 'all' || options.scenario === 'update-js') {
|
|
console.log(` [Scenario: Update N->N+1 (1-line app.js)] running ${options.runs} iterations...`);
|
|
const updJsRuns = [];
|
|
for (let r = 0; r < options.runs; r++) {
|
|
process.stdout.write(` run ${r + 1}/${options.runs}... `);
|
|
const res = await runUpdateScenario({
|
|
browserType: bType,
|
|
server,
|
|
proxy,
|
|
profile: prof,
|
|
updateKind: 'js',
|
|
trees,
|
|
});
|
|
updJsRuns.push(res);
|
|
console.log(`done (${res.wireBytes} bytes, ${res.requestCount} reqs, dur=${res.duration}ms)`);
|
|
}
|
|
|
|
results.browsers[bName][prof].updateJs = {
|
|
requestCount: summarizeList(updJsRuns.map((r) => r.requestCount)),
|
|
wireBytes: summarizeList(updJsRuns.map((r) => r.wireBytes)),
|
|
duration: summarizeList(updJsRuns.map((r) => r.duration)),
|
|
wireBytesByType: updJsRuns[0]?.wireBytesByType || {},
|
|
};
|
|
}
|
|
|
|
// 5. Update N -> N+1 (CSS)
|
|
if (options.scenario === 'all' || options.scenario === 'update-css') {
|
|
console.log(` [Scenario: Update N->N+1 (1-line theme-glass.css)] running ${options.runs} iterations...`);
|
|
const updCssRuns = [];
|
|
for (let r = 0; r < options.runs; r++) {
|
|
process.stdout.write(` run ${r + 1}/${options.runs}... `);
|
|
const res = await runUpdateScenario({
|
|
browserType: bType,
|
|
server,
|
|
proxy,
|
|
profile: prof,
|
|
updateKind: 'css',
|
|
trees,
|
|
});
|
|
updCssRuns.push(res);
|
|
console.log(`done (${res.wireBytes} bytes, ${res.requestCount} reqs, dur=${res.duration}ms)`);
|
|
}
|
|
|
|
results.browsers[bName][prof].updateCss = {
|
|
requestCount: summarizeList(updCssRuns.map((r) => r.requestCount)),
|
|
wireBytes: summarizeList(updCssRuns.map((r) => r.wireBytes)),
|
|
duration: summarizeList(updCssRuns.map((r) => r.duration)),
|
|
wireBytesByType: updCssRuns[0]?.wireBytesByType || {},
|
|
};
|
|
}
|
|
|
|
await proxy.close();
|
|
}
|
|
}
|
|
} finally {
|
|
await server.stop();
|
|
rmSync(tmpRoot, { recursive: true, force: true });
|
|
}
|
|
|
|
// Write results JSON
|
|
const now = new Date();
|
|
const dateStr = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`;
|
|
const outPath = options.out || path.join(REPO_ROOT, 'perf', 'results', `baseline-${dateStr}.json`);
|
|
mkdirSync(path.dirname(outPath), { recursive: true });
|
|
writeFileSync(outPath, JSON.stringify(results, null, 2));
|
|
console.log(`\nBaseline results saved to: ${outPath}`);
|
|
|
|
// Print Summary Table
|
|
printSummaryTable(results);
|
|
|
|
// Compare mode if requested
|
|
if (options.compare) {
|
|
compareResults(options.compare, results);
|
|
}
|
|
}
|
|
|
|
function printSummaryTable(results) {
|
|
console.log('\n============================= HEADLINE BASELINE SUMMARY =============================');
|
|
for (const [browser, profiles] of Object.entries(results.browsers)) {
|
|
for (const [profile, scenarios] of Object.entries(profiles)) {
|
|
console.log(`\n[Browser: ${browser} | Profile: ${profile}]`);
|
|
console.log('| Scenario | Req Count | Wire Bytes | Boot / Ready | SW Dur / Reload |');
|
|
console.log('|---------------------------|-----------|------------|--------------|-----------------|');
|
|
|
|
if (scenarios.cold) {
|
|
const c = scenarios.cold;
|
|
console.log(
|
|
`| Cold First Load | ${String(c.requestCount.median).padEnd(9)} | ${formatBytes(c.wireBytes.median).padEnd(10)} | ${String(c.bootDone.median + 'ms').padEnd(12)} | ${String(c.swInstallDuration.median + 'ms').padEnd(15)} |`
|
|
);
|
|
}
|
|
if (scenarios.warm) {
|
|
const w = scenarios.warm;
|
|
console.log(
|
|
`| Warm Reload (under SW) | ${String(w.requestCount.median).padEnd(9)} | ${formatBytes(w.wireBytes.median).padEnd(10)} | ${String(w.bootDone.median + 'ms').padEnd(12)} | ${'-'.padEnd(15)} |`
|
|
);
|
|
}
|
|
if (scenarios.offline) {
|
|
const o = scenarios.offline;
|
|
console.log(
|
|
`| Offline Reload (under SW) | ${String(o.requestCount.median).padEnd(9)} | ${formatBytes(o.wireBytes.median).padEnd(10)} | ${String(o.bootDone.median + 'ms').padEnd(12)} | ${String(o.reloadDuration.median + 'ms').padEnd(15)} |`
|
|
);
|
|
}
|
|
if (scenarios.updateJs) {
|
|
const u = scenarios.updateJs;
|
|
console.log(
|
|
`| Update N->N+1 (1-line JS) | ${String(u.requestCount.median).padEnd(9)} | ${formatBytes(u.wireBytes.median).padEnd(10)} | ${'-'.padEnd(12)} | ${String(u.duration.median + 'ms').padEnd(15)} |`
|
|
);
|
|
}
|
|
if (scenarios.updateCss) {
|
|
const u = scenarios.updateCss;
|
|
console.log(
|
|
`| Update N->N+1 (1-line CSS)| ${String(u.requestCount.median).padEnd(9)} | ${formatBytes(u.wireBytes.median).padEnd(10)} | ${'-'.padEnd(12)} | ${String(u.duration.median + 'ms').padEnd(15)} |`
|
|
);
|
|
}
|
|
}
|
|
}
|
|
console.log('\n=====================================================================================');
|
|
}
|
|
|
|
function formatBytes(bytes) {
|
|
if (bytes === null || bytes === undefined) return 'N/A';
|
|
if (bytes >= 1024 * 1024) return (bytes / (1024 * 1024)).toFixed(2) + ' MB';
|
|
if (bytes >= 1024) return (bytes / 1024).toFixed(1) + ' KB';
|
|
return bytes + ' B';
|
|
}
|
|
|
|
function compareResults(baselinePath, currentResults) {
|
|
console.log(`\n================ Phase 5 Comparison against: ${baselinePath} ================`);
|
|
if (!existsSync(baselinePath)) {
|
|
console.error(`Error: Baseline file not found: ${baselinePath}`);
|
|
return;
|
|
}
|
|
const base = JSON.parse(readFileSync(baselinePath, 'utf8'));
|
|
|
|
for (const [browser, profiles] of Object.entries(currentResults.browsers)) {
|
|
for (const [profile, scenarios] of Object.entries(profiles)) {
|
|
console.log(`\n[Comparison: ${browser} / ${profile}]`);
|
|
console.log('| Metric | Baseline | Current | Delta (%) |');
|
|
console.log('|------------------------------------|-------------|-------------|--------------|');
|
|
|
|
const baseScen = base.browsers?.[browser]?.[profile];
|
|
if (!baseScen) {
|
|
console.log('| (No matching baseline data) | | | |');
|
|
continue;
|
|
}
|
|
|
|
const rows = [
|
|
['Cold: Wire Bytes', baseScen.cold?.wireBytes?.median, scenarios.cold?.wireBytes?.median, true],
|
|
['Cold: Boot Done (ms)', baseScen.cold?.bootDone?.median, scenarios.cold?.bootDone?.median, false],
|
|
['Cold: Requests', baseScen.cold?.requestCount?.median, scenarios.cold?.requestCount?.median, false],
|
|
['Warm: Boot Done (ms)', baseScen.warm?.bootDone?.median, scenarios.warm?.bootDone?.median, false],
|
|
['Update (1-line JS): Wire Bytes', baseScen.updateJs?.wireBytes?.median, scenarios.updateJs?.wireBytes?.median, true],
|
|
['Update (1-line JS): Time (ms)', baseScen.updateJs?.duration?.median, scenarios.updateJs?.duration?.median, false],
|
|
['Update (1-line CSS): Wire Bytes', baseScen.updateCss?.wireBytes?.median, scenarios.updateCss?.wireBytes?.median, true],
|
|
['Update (1-line CSS): Time (ms)', baseScen.updateCss?.duration?.median, scenarios.updateCss?.duration?.median, false],
|
|
];
|
|
|
|
for (const [name, bVal, cVal, isBytes] of rows) {
|
|
if (bVal === undefined || cVal === undefined) continue;
|
|
const bStr = isBytes ? formatBytes(bVal) : bVal !== null ? `${bVal}` : 'N/A';
|
|
const cStr = isBytes ? formatBytes(cVal) : cVal !== null ? `${cVal}` : 'N/A';
|
|
let deltaStr = '-';
|
|
if (typeof bVal === 'number' && typeof cVal === 'number' && bVal > 0) {
|
|
const pct = (((cVal - bVal) / bVal) * 100).toFixed(1);
|
|
const sign = cVal >= bVal ? '+' : '';
|
|
deltaStr = `${sign}${pct}%`;
|
|
}
|
|
console.log(`| ${name.padEnd(34)} | ${bStr.padEnd(11)} | ${cStr.padEnd(11)} | ${deltaStr.padEnd(12)} |`);
|
|
}
|
|
}
|
|
}
|
|
console.log('==============================================================================');
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error('Benchmark fatal error:', err);
|
|
process.exit(1);
|
|
});
|