josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add live server performance monitoring

Browse Source
This commit is contained in:
Jonathan Sykes
2026-10-14 02:43:20 +08:00
parent 4884589f36
commit e9a597c415
5 changed files with 159 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
frontend/admin.html
View File

@@ -740,6 +740,16 @@
<div class="card"><h2>Most played</h2><div id="analyticsTop"></div></div>
</section>
<section id="p-ops" class="panel">
<div class="row"><div><h1 class="page">Performance</h1><p class="lead">Route latency, service queues, process health and storage headroom.</p></div><span class="grow"></span><button class="btn" id="opsRefresh" type="button">Refresh</button></div>
<p id="opsStatus" class="msg" role="status" aria-live="polite"></p>
<div class="analytics-grid" id="opsCards"></div>
<div class="card"><h2>Route health</h2><div class="scroll"><table class="tbl" id="opsRoutes"></table></div></div>
<div class="card"><h2>Storage volumes</h2><div id="opsVolumes"></div></div>
<div class="card"><h2>24-hour p95 history</h2><div id="opsHistory" class="analytics-grid"></div></div>
<div id="opsAlerts" aria-live="polite"></div>
</section>
<section id="p-deep-search" class="panel deep-search">
<h1 class="page">Deep search</h1>
<p class="lead">Build the local video index from YouTube searches. Runs are owned by the server and continue if this page closes.</p>
@@ -841,11 +851,12 @@
history: '<path d="M3 12a9 9 0 109-9 9 9 0 00-6.4 2.6L3 8"/><path d="M3 3v5h5M12 7v5l3 2"/>',
backups: '<path d="M4 5h16v15H4zM8 5V3h8v2M8 11h8M8 15h5"/>',
analytics: '<path d="M4 20h16M7 16V9M12 16V4M17 16v-5"/>',
ops: '<path d="M4 19V5m0 14h16M7 15l3-4 3 2 5-7"/><circle cx="18" cy="6" r="1.5"/>',
'deep-search': '<circle cx="10.5" cy="10.5" r="6.5"/><path d="m15.5 15.5 5 5M8 10.5h5M10.5 8v5"/>',
audit: '<path d="M5 4h14v16H5zM8 8h8M8 12h8M8 16h5"/>',
access: '<circle cx="8" cy="15" r="4"/><path d="M11 12l9-9M16 7l3 3M14 9l2 2"/>',
};
const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['lyric-search', 'Lyrics search'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['lyric-search', 'Lyrics search'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['ops', 'Performance'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
let tab = 'editor';
function buildTabs() {
const html = TABS.map(([k, l]) => `<button class="tab" role="tab" data-tab="${k}" aria-selected="false"><svg viewBox="0 0 24 24">${ICONS[k]}</svg><span>${l}</span>${k === 'reports' ? '<span class="badge hidden" data-badge>0</span>' : ''}</button>`).join('');
@@ -870,6 +881,7 @@
if (name === 'library') { loadUploads(); loadBulkLibrary(); loadBulkJobs(); }
if (name === 'access') loadTokens();
if (name === 'analytics') loadAnalytics();
if (name === 'ops') loadOpsMetrics();
if (name === 'deep-search') loadDeepSearch();
if (name === 'audit') loadAudit();
window.scrollTo({ top: 0 });
@@ -937,6 +949,30 @@
await Promise.all([loadCollections(), loadMetadata()]);
} catch (error) { $('analyticsStatus').textContent = error.message || 'Could not load server analytics.'; }
}
async function loadOpsMetrics() {
const status = $('opsStatus'); if (!status) return;
status.textContent = 'Sampling service health…';
try {
const { current, history = [] } = await api('/api/admin/ops/metrics');
const cards = [
['In flight', current.inflight], ['Memory RSS', bytes(current.rssBytes)], ['CPU', `${Number(current.cpuPercent || 0).toFixed(1)}%`],
['yt-dlp pool', current.ytdlp ? `${current.ytdlp.busy}/${current.ytdlp.workers} busy · ${current.ytdlp.queued} queued` : 'Unavailable'],
['Media queue', current.media ? `${current.media.queued} queued · ${current.media.active ? '1 active' : 'idle'}` : 'Unavailable'],
['Search cache', current.cache ? `${current.cache.hits} hits · ${current.cache.misses} misses` : 'Unavailable'],
];
$('opsCards').innerHTML = cards.map(([name,value]) => `<div class="metric"><span>${esc(name)}</span><b>${esc(value)}</b></div>`).join('');
const rows = Object.entries(current.routes || {}).sort((a,b)=>b[1].requests-a[1].requests);
$('opsRoutes').innerHTML = `<tr><th>Route</th><th>Requests</th><th>p50</th><th>p95</th><th>p99</th><th>Errors</th></tr>` + (rows.map(([path,r])=>`<tr><td><code>${esc(path)}</code></td><td>${r.requests}</td><td>${r.p50} ms</td><td>${r.p95} ms</td><td>${r.p99} ms</td><td>${r.errors} (${(r.errorRate*100).toFixed(1)}%)</td></tr>`).join('') || '<tr><td colspan="6" class="muted">No requests recorded yet.</td></tr>');
$('opsVolumes').innerHTML = (current.volumes || []).map(v=>`<p><b>${esc(v.name)}</b> · ${v.unavailable?'Unavailable':`${bytes(v.freeBytes)} free of ${bytes(v.totalBytes)}`}</p>`).join('');
const points = history.slice(-24).map(s=>Math.max(0,...Object.values(s.routes||{}).map(r=>r.p95)));
const maxPoint = Math.max(1,...points), poly = points.map((v,i)=>`${points.length < 2 ? 0 : i*100/(points.length-1)},${36-v/maxPoint*32}`).join(' ');
$('opsHistory').innerHTML = `<div class="metric" style="grid-column:1/-1"><span>Peak route p95 · last ${points.length} snapshots</span><b>${maxPoint} ms</b><svg role="img" aria-label="Peak p95 latency sparkline" viewBox="0 0 100 40" preserveAspectRatio="none" style="width:100%;height:64px;stroke:var(--a1);fill:none;stroke-width:2"><polyline points="${poly}" vector-effect="non-scaling-stroke"/></svg></div>`;
const alerts = [];
for (const [path,r] of rows) if (r.p95 > 5000 || r.errorRate > .1) alerts.push(`<div class="card" role="status"><b>Attention:</b> ${esc(path)} p95 ${r.p95} ms · errors ${(r.errorRate*100).toFixed(1)}%</div>`);
$('opsAlerts').innerHTML = alerts.join(''); status.textContent = `Updated ${new Date(current.at).toLocaleTimeString()}`;
} catch (error) { status.textContent = error.message || 'Monitoring unavailable.'; }
}
$('opsRefresh')?.addEventListener('click', loadOpsMetrics);
async function loadCollections() {
try {
const j = await api('/api/admin/collections'); if (!j.ok) return;

85
server/request-monitor.js Normal file
View File

@@ -0,0 +1,85 @@
import { appendFile, mkdir, readFile } from 'node:fs/promises';
import { dirname } from 'node:path';
import { statfs } from 'node:fs/promises';
const routeKey = (path) => path
.replace(/\/api\/media\/[^/]+/g, '/api/media/:id')
.replace(/\/api\/download\/[^/]+/g, '/api/download/:id')
.replace(/\/api\/admin\/uploads\/[^/]+/g, '/api/admin/uploads/:id')
.replace(/\?.*$/, '');
const percentile = (values, p) => values.length ? values[Math.min(values.length - 1, Math.ceil(values.length * p) - 1)] : 0;
export function createRequestMonitor({ snapshotFile = null, diskPaths = [], adapters = {}, now = () => Date.now(), snapshotIntervalMs = 60_000 } = {}) {
const routes = new Map();
const snapshots = [];
let inflight = 0;
let lastCpu = process.cpuUsage();
let lastCpuAt = now();
let snapshotTimer;
async function sample() {
const cpuNow = process.cpuUsage();
const elapsed = Math.max(1, now() - lastCpuAt) * 1000;
const cpuPct = Math.min(100 * (cpuNow.user + cpuNow.system - lastCpu.user - lastCpu.system) / elapsed, 100);
lastCpu = cpuNow; lastCpuAt = now();
const volumes = await Promise.all(diskPaths.map(async ({ name, path }) => {
try { const fs = await statfs(path); return { name, freeBytes: Number(fs.bavail) * Number(fs.bsize), totalBytes: Number(fs.blocks) * Number(fs.bsize) }; }
catch { return { name, unavailable: true }; }
}));
const routeMetrics = Object.fromEntries([...routes].map(([key, v]) => {
const sorted = [...v.latencies].sort((a, b) => a - b);
return [key, { p50: percentile(sorted, .5), p95: percentile(sorted, .95), p99: percentile(sorted, .99), requests: v.requests, errors: v.errors, errorRate: v.requests ? v.errors / v.requests : 0 }];
}));
const [ytdlp, media, transcription, cache] = await Promise.all([
adapters.ytdlp?.() ?? null, adapters.media?.() ?? null, adapters.transcription?.() ?? null, adapters.cache?.() ?? null,
]);
return {
at: now(), routes: routeMetrics, inflight, rssBytes: process.memoryUsage().rss, cpuPercent: cpuPct,
ytdlp, media, transcription, cache, volumes,
};
}
async function snapshot() {
const value = await sample(); snapshots.push(value);
while (snapshots.length && now() - snapshots[0].at > 24 * 3600_000) snapshots.shift();
if (snapshotFile) {
try { await mkdir(dirname(snapshotFile), { recursive: true }); await appendFile(snapshotFile, JSON.stringify(value) + '\n'); }
catch (error) { console.warn('[monitor] snapshot write failed:', error.message); }
}
return value;
}
async function history() {
const cutoff = now() - 24 * 3600_000;
const out = snapshots.filter(x => x.at >= cutoff);
if (snapshotFile) {
try {
const persisted = (await readFile(snapshotFile, 'utf8')).split('\n').filter(Boolean).map(line => { try { return JSON.parse(line); } catch { return null; } }).filter(x => x?.at >= cutoff);
return [...persisted, ...out].slice(-1440);
} catch { /* no file yet */ }
}
return out.slice(-1440);
}
function middleware() {
return async (c, next) => {
const started = now(); inflight++;
let status = 500;
try { await next(); status = c.res.status; }
finally {
inflight--;
const key = routeKey(c.req.path);
let row = routes.get(key);
if (!row) { row = { latencies: [], requests: 0, errors: 0 }; routes.set(key, row); }
row.requests++; if (status >= 500) row.errors++;
row.latencies.push(Math.max(0, now() - started)); if (row.latencies.length > 1000) row.latencies.shift();
}
};
}
if (snapshotIntervalMs > 0) { snapshotTimer = setInterval(() => snapshot().catch(() => {}), snapshotIntervalMs); snapshotTimer.unref?.(); }
return { middleware, sample, snapshot, history, close: () => clearInterval(snapshotTimer) };
}
export function registerRequestMonitorRoutes(app, { adminAuth, monitor }) {
app.get('/api/admin/ops/metrics', adminAuth, async c => c.json({ ok: true, current: await monitor.sample(), history: await monitor.history() }, 200, { 'Cache-Control': 'no-store' }));
}

19
server/request-monitor.test.js Normal file
View File

@@ -0,0 +1,19 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { createRequestMonitor } from './request-monitor.js';
test('request monitor records route latency, status errors, and current process health', async () => {
let t = 100;
const monitor = createRequestMonitor({ now: () => t, snapshotIntervalMs: 0, adapters: { media: () => ({ queued: 2 }) } });
const middleware = monitor.middleware();
const ctx = (path, status) => ({ req: { path }, res: { status } });
await middleware(ctx('/api/media/abc/status', 200), async () => { t += 8; });
await middleware(ctx('/api/media/xyz/status', 503), async () => { t += 12; });
const sample = await monitor.sample();
assert.equal(sample.routes['/api/media/:id/status'].requests, 2);
assert.equal(sample.routes['/api/media/:id/status'].errors, 1);
assert.equal(sample.routes['/api/media/:id/status'].p50, 8);
assert.equal(sample.media.queued, 2);
assert.ok(sample.rssBytes > 0);
monitor.close();
});

18
server/server.js
View File

@@ -71,6 +71,7 @@ import { createYtdlpPool } from './ytdlp-pool.js';
import { createWarmQueue } from './warm-queue.js';
import { registerDeepSearchRoutes } from './deep-search.js';
import { dirname, join as pathJoin } from 'node:path';
import { createRequestMonitor, registerRequestMonitorRoutes } from './request-monitor.js';
// A media proxy must not die because one client's stream hit an edge case
// (see /api/play cancel()): log and keep serving instead of crash-looping.
@@ -346,6 +347,17 @@ function channelToUrl(c) {
const app = new Hono();
app.use('*', logger());
const requestMonitor = createRequestMonitor({
snapshotFile: pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'ops-metrics.jsonl'),
diskPaths: [{ name: 'app-data', path: dirname(process.env.DB_PATH || './data/ytplayer.db') }, { name: 'media', path: process.env.MEDIA_DIR || './data/media' }],
adapters: {
ytdlp: () => ytdlpPool ? ytdlpPool.stats() : null,
media: () => media?.stats ? media.stats() : null,
transcription: () => notesDb.transcriptionStats?.() || null,
cache: () => ({ ...searchCacheCounters, memoryEntries: searchCache.size }),
},
});
app.use('*', requestMonitor.middleware());
registerCatalogRoutes(app, { resolveListener: recommendationListener });
// ============================================================================
@@ -383,6 +395,7 @@ const SEARCH_CACHE_TTL_MS = 10 * 60_000; // fresh: answered from memory
const SEARCH_CACHE_STALE_MS = 15 * 24 * 3600_000; // persistent copy: stale-while-revalidate for 15 days
const searchCache = new Map(); // lowercased q -> { yt, fetchedAt } (YouTube cards only)
const inflightSearches = new Map(); // lowercased q -> Promise<{ yt, youtubeError? }>
const searchCacheCounters = { hits: 0, misses: 0 };
// The YouTube side of one search (up to 200 cards), shared by every caller asking
// the same thing at the same time, and written to memory + the persistent cache.
@@ -452,11 +465,13 @@ app.get('/api/search', async (c) => {
if (row) { hit = { yt: row.results, fetchedAt: row.fetchedAt }; searchCache.set(key, hit); }
}
const age = hit ? Date.now() - hit.fetchedAt : Infinity;
if (hit && age < SEARCH_CACHE_TTL_MS) return answer(hit.yt, { fetchedAt: hit.fetchedAt });
if (hit && age < SEARCH_CACHE_TTL_MS) { searchCacheCounters.hits++; return answer(hit.yt, { fetchedAt: hit.fetchedAt }); }
if (hit && age < SEARCH_CACHE_STALE_MS) {
searchCacheCounters.hits++;
fetchYoutube(q).catch(() => { /* keep serving the stale copy */ });
return answer(hit.yt, { stale: true, fetchedAt: hit.fetchedAt });
}
searchCacheCounters.misses++;
}
try {
const { yt, fetchedAt } = await fetchYoutube(q);
@@ -2340,6 +2355,7 @@ const uploads = registerUploadRoutes(app, {
parseLrc,
sanitizeLyrics,
});
registerRequestMonitorRoutes(app, { adminAuth: notes.requireAdminOrToken, monitor: requestMonitor });
const isUpload = (id) => uploads.isUploadId(id);
// ============================================================================

2
server/ytdlp-pool.js
View File

@@ -108,5 +108,5 @@ export function createYtdlpPool({ ytdlpPath = '', size = 2, timeoutMs = 60_000,
for (const w of workers) if (!w.busy) retire(w);
}
return { run, close, recycle, get disabled() { return disabled; } };
return { run, close, recycle, stats: () => ({ queued: queue.length, workers: workers.length, busy: workers.filter(w => !!w.busy).length, disabled }), get disabled() { return disabled; } };
}