From e9a597c4158fed288da462436608d3df24b6e4bc Mon Sep 17 00:00:00 2001
From: Jonathan Sykes <telemagnadon@gmail.com>
Date: Wed, 14 Oct 2026 02:43:20 +0800
Subject: [PATCH] Add live server performance monitoring

---
 frontend/admin.html            | 38 ++++++++++++++-
 server/request-monitor.js      | 85 ++++++++++++++++++++++++++++++++++
 server/request-monitor.test.js | 19 ++++++++
 server/server.js               | 18 ++++++-
 server/ytdlp-pool.js           |  2 +-
 5 files changed, 159 insertions(+), 3 deletions(-)
 create mode 100644 server/request-monitor.js
 create mode 100644 server/request-monitor.test.js

diff --git a/frontend/admin.html b/frontend/admin.html
index e1dc8f3..d4940c7 100644
--- a/frontend/admin.html
+++ b/frontend/admin.html
@@ -740,6 +740,16 @@
       <div class="card"><h2>Most played</h2><div id="analyticsTop"></div></div>
     </section>
 
+    <section id="p-ops" class="panel">
+      <div class="row"><div><h1 class="page">Performance</h1><p class="lead">Route latency, service queues, process health and storage headroom.</p></div><span class="grow"></span><button class="btn" id="opsRefresh" type="button">Refresh</button></div>
+      <p id="opsStatus" class="msg" role="status" aria-live="polite"></p>
+      <div class="analytics-grid" id="opsCards"></div>
+      <div class="card"><h2>Route health</h2><div class="scroll"><table class="tbl" id="opsRoutes"></table></div></div>
+      <div class="card"><h2>Storage volumes</h2><div id="opsVolumes"></div></div>
+      <div class="card"><h2>24-hour p95 history</h2><div id="opsHistory" class="analytics-grid"></div></div>
+      <div id="opsAlerts" aria-live="polite"></div>
+    </section>
+
     <section id="p-deep-search" class="panel deep-search">
       <h1 class="page">Deep search</h1>
       <p class="lead">Build the local video index from YouTube searches. Runs are owned by the server and continue if this page closes.</p>
@@ -841,11 +851,12 @@
     history: '<path d="M3 12a9 9 0 109-9 9 9 0 00-6.4 2.6L3 8"/><path d="M3 3v5h5M12 7v5l3 2"/>',
     backups: '<path d="M4 5h16v15H4zM8 5V3h8v2M8 11h8M8 15h5"/>',
     analytics: '<path d="M4 20h16M7 16V9M12 16V4M17 16v-5"/>',
+    ops: '<path d="M4 19V5m0 14h16M7 15l3-4 3 2 5-7"/><circle cx="18" cy="6" r="1.5"/>',
     'deep-search': '<circle cx="10.5" cy="10.5" r="6.5"/><path d="m15.5 15.5 5 5M8 10.5h5M10.5 8v5"/>',
     audit: '<path d="M5 4h14v16H5zM8 8h8M8 12h8M8 16h5"/>',
     access: '<circle cx="8" cy="15" r="4"/><path d="M11 12l9-9M16 7l3 3M14 9l2 2"/>',
   };
-  const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['lyric-search', 'Lyrics search'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
+  const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['lyric-search', 'Lyrics search'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['ops', 'Performance'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
   let tab = 'editor';
   function buildTabs() {
     const html = TABS.map(([k, l]) => `<button class="tab" role="tab" data-tab="${k}" aria-selected="false"><svg viewBox="0 0 24 24">${ICONS[k]}</svg><span>${l}</span>${k === 'reports' ? '<span class="badge hidden" data-badge>0</span>' : ''}</button>`).join('');
@@ -870,6 +881,7 @@
     if (name === 'library') { loadUploads(); loadBulkLibrary(); loadBulkJobs(); }
     if (name === 'access') loadTokens();
     if (name === 'analytics') loadAnalytics();
+    if (name === 'ops') loadOpsMetrics();
     if (name === 'deep-search') loadDeepSearch();
     if (name === 'audit') loadAudit();
     window.scrollTo({ top: 0 });
@@ -937,6 +949,30 @@
       await Promise.all([loadCollections(), loadMetadata()]);
     } catch (error) { $('analyticsStatus').textContent = error.message || 'Could not load server analytics.'; }
   }
+  async function loadOpsMetrics() {
+    const status = $('opsStatus'); if (!status) return;
+    status.textContent = 'Sampling service health…';
+    try {
+      const { current, history = [] } = await api('/api/admin/ops/metrics');
+      const cards = [
+        ['In flight', current.inflight], ['Memory RSS', bytes(current.rssBytes)], ['CPU', `${Number(current.cpuPercent || 0).toFixed(1)}%`],
+        ['yt-dlp pool', current.ytdlp ? `${current.ytdlp.busy}/${current.ytdlp.workers} busy · ${current.ytdlp.queued} queued` : 'Unavailable'],
+        ['Media queue', current.media ? `${current.media.queued} queued · ${current.media.active ? '1 active' : 'idle'}` : 'Unavailable'],
+        ['Search cache', current.cache ? `${current.cache.hits} hits · ${current.cache.misses} misses` : 'Unavailable'],
+      ];
+      $('opsCards').innerHTML = cards.map(([name,value]) => `<div class="metric"><span>${esc(name)}</span><b>${esc(value)}</b></div>`).join('');
+      const rows = Object.entries(current.routes || {}).sort((a,b)=>b[1].requests-a[1].requests);
+      $('opsRoutes').innerHTML = `<tr><th>Route</th><th>Requests</th><th>p50</th><th>p95</th><th>p99</th><th>Errors</th></tr>` + (rows.map(([path,r])=>`<tr><td><code>${esc(path)}</code></td><td>${r.requests}</td><td>${r.p50} ms</td><td>${r.p95} ms</td><td>${r.p99} ms</td><td>${r.errors} (${(r.errorRate*100).toFixed(1)}%)</td></tr>`).join('') || '<tr><td colspan="6" class="muted">No requests recorded yet.</td></tr>');
+      $('opsVolumes').innerHTML = (current.volumes || []).map(v=>`<p><b>${esc(v.name)}</b> · ${v.unavailable?'Unavailable':`${bytes(v.freeBytes)} free of ${bytes(v.totalBytes)}`}</p>`).join('');
+      const points = history.slice(-24).map(s=>Math.max(0,...Object.values(s.routes||{}).map(r=>r.p95)));
+      const maxPoint = Math.max(1,...points), poly = points.map((v,i)=>`${points.length < 2 ? 0 : i*100/(points.length-1)},${36-v/maxPoint*32}`).join(' ');
+      $('opsHistory').innerHTML = `<div class="metric" style="grid-column:1/-1"><span>Peak route p95 · last ${points.length} snapshots</span><b>${maxPoint} ms</b><svg role="img" aria-label="Peak p95 latency sparkline" viewBox="0 0 100 40" preserveAspectRatio="none" style="width:100%;height:64px;stroke:var(--a1);fill:none;stroke-width:2"><polyline points="${poly}" vector-effect="non-scaling-stroke"/></svg></div>`;
+      const alerts = [];
+      for (const [path,r] of rows) if (r.p95 > 5000 || r.errorRate > .1) alerts.push(`<div class="card" role="status"><b>Attention:</b> ${esc(path)} p95 ${r.p95} ms · errors ${(r.errorRate*100).toFixed(1)}%</div>`);
+      $('opsAlerts').innerHTML = alerts.join(''); status.textContent = `Updated ${new Date(current.at).toLocaleTimeString()}`;
+    } catch (error) { status.textContent = error.message || 'Monitoring unavailable.'; }
+  }
+  $('opsRefresh')?.addEventListener('click', loadOpsMetrics);
   async function loadCollections() {
     try {
       const j = await api('/api/admin/collections'); if (!j.ok) return;
diff --git a/server/request-monitor.js b/server/request-monitor.js
new file mode 100644
index 0000000..8f29023
--- /dev/null
+++ b/server/request-monitor.js
@@ -0,0 +1,85 @@
+import { appendFile, mkdir, readFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { statfs } from 'node:fs/promises';
+
+const routeKey = (path) => path
+  .replace(/\/api\/media\/[^/]+/g, '/api/media/:id')
+  .replace(/\/api\/download\/[^/]+/g, '/api/download/:id')
+  .replace(/\/api\/admin\/uploads\/[^/]+/g, '/api/admin/uploads/:id')
+  .replace(/\?.*$/, '');
+const percentile = (values, p) => values.length ? values[Math.min(values.length - 1, Math.ceil(values.length * p) - 1)] : 0;
+
+export function createRequestMonitor({ snapshotFile = null, diskPaths = [], adapters = {}, now = () => Date.now(), snapshotIntervalMs = 60_000 } = {}) {
+  const routes = new Map();
+  const snapshots = [];
+  let inflight = 0;
+  let lastCpu = process.cpuUsage();
+  let lastCpuAt = now();
+  let snapshotTimer;
+
+  async function sample() {
+    const cpuNow = process.cpuUsage();
+    const elapsed = Math.max(1, now() - lastCpuAt) * 1000;
+    const cpuPct = Math.min(100 * (cpuNow.user + cpuNow.system - lastCpu.user - lastCpu.system) / elapsed, 100);
+    lastCpu = cpuNow; lastCpuAt = now();
+    const volumes = await Promise.all(diskPaths.map(async ({ name, path }) => {
+      try { const fs = await statfs(path); return { name, freeBytes: Number(fs.bavail) * Number(fs.bsize), totalBytes: Number(fs.blocks) * Number(fs.bsize) }; }
+      catch { return { name, unavailable: true }; }
+    }));
+    const routeMetrics = Object.fromEntries([...routes].map(([key, v]) => {
+      const sorted = [...v.latencies].sort((a, b) => a - b);
+      return [key, { p50: percentile(sorted, .5), p95: percentile(sorted, .95), p99: percentile(sorted, .99), requests: v.requests, errors: v.errors, errorRate: v.requests ? v.errors / v.requests : 0 }];
+    }));
+    const [ytdlp, media, transcription, cache] = await Promise.all([
+      adapters.ytdlp?.() ?? null, adapters.media?.() ?? null, adapters.transcription?.() ?? null, adapters.cache?.() ?? null,
+    ]);
+    return {
+      at: now(), routes: routeMetrics, inflight, rssBytes: process.memoryUsage().rss, cpuPercent: cpuPct,
+      ytdlp, media, transcription, cache, volumes,
+    };
+  }
+
+  async function snapshot() {
+    const value = await sample(); snapshots.push(value);
+    while (snapshots.length && now() - snapshots[0].at > 24 * 3600_000) snapshots.shift();
+    if (snapshotFile) {
+      try { await mkdir(dirname(snapshotFile), { recursive: true }); await appendFile(snapshotFile, JSON.stringify(value) + '\n'); }
+      catch (error) { console.warn('[monitor] snapshot write failed:', error.message); }
+    }
+    return value;
+  }
+  async function history() {
+    const cutoff = now() - 24 * 3600_000;
+    const out = snapshots.filter(x => x.at >= cutoff);
+    if (snapshotFile) {
+      try {
+        const persisted = (await readFile(snapshotFile, 'utf8')).split('\n').filter(Boolean).map(line => { try { return JSON.parse(line); } catch { return null; } }).filter(x => x?.at >= cutoff);
+        return [...persisted, ...out].slice(-1440);
+      } catch { /* no file yet */ }
+    }
+    return out.slice(-1440);
+  }
+
+  function middleware() {
+    return async (c, next) => {
+      const started = now(); inflight++;
+      let status = 500;
+      try { await next(); status = c.res.status; }
+      finally {
+        inflight--;
+        const key = routeKey(c.req.path);
+        let row = routes.get(key);
+        if (!row) { row = { latencies: [], requests: 0, errors: 0 }; routes.set(key, row); }
+        row.requests++; if (status >= 500) row.errors++;
+        row.latencies.push(Math.max(0, now() - started)); if (row.latencies.length > 1000) row.latencies.shift();
+      }
+    };
+  }
+
+  if (snapshotIntervalMs > 0) { snapshotTimer = setInterval(() => snapshot().catch(() => {}), snapshotIntervalMs); snapshotTimer.unref?.(); }
+  return { middleware, sample, snapshot, history, close: () => clearInterval(snapshotTimer) };
+}
+
+export function registerRequestMonitorRoutes(app, { adminAuth, monitor }) {
+  app.get('/api/admin/ops/metrics', adminAuth, async c => c.json({ ok: true, current: await monitor.sample(), history: await monitor.history() }, 200, { 'Cache-Control': 'no-store' }));
+}
diff --git a/server/request-monitor.test.js b/server/request-monitor.test.js
new file mode 100644
index 0000000..8f9a07a
--- /dev/null
+++ b/server/request-monitor.test.js
@@ -0,0 +1,19 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createRequestMonitor } from './request-monitor.js';
+
+test('request monitor records route latency, status errors, and current process health', async () => {
+  let t = 100;
+  const monitor = createRequestMonitor({ now: () => t, snapshotIntervalMs: 0, adapters: { media: () => ({ queued: 2 }) } });
+  const middleware = monitor.middleware();
+  const ctx = (path, status) => ({ req: { path }, res: { status } });
+  await middleware(ctx('/api/media/abc/status', 200), async () => { t += 8; });
+  await middleware(ctx('/api/media/xyz/status', 503), async () => { t += 12; });
+  const sample = await monitor.sample();
+  assert.equal(sample.routes['/api/media/:id/status'].requests, 2);
+  assert.equal(sample.routes['/api/media/:id/status'].errors, 1);
+  assert.equal(sample.routes['/api/media/:id/status'].p50, 8);
+  assert.equal(sample.media.queued, 2);
+  assert.ok(sample.rssBytes > 0);
+  monitor.close();
+});
diff --git a/server/server.js b/server/server.js
index 4340f7c..963b9f1 100644
--- a/server/server.js
+++ b/server/server.js
@@ -71,6 +71,7 @@ import { createYtdlpPool } from './ytdlp-pool.js';
 import { createWarmQueue } from './warm-queue.js';
 import { registerDeepSearchRoutes } from './deep-search.js';
 import { dirname, join as pathJoin } from 'node:path';
+import { createRequestMonitor, registerRequestMonitorRoutes } from './request-monitor.js';
 
 // A media proxy must not die because one client's stream hit an edge case
 // (see /api/play cancel()): log and keep serving instead of crash-looping.
@@ -346,6 +347,17 @@ function channelToUrl(c) {
 
 const app = new Hono();
 app.use('*', logger());
+const requestMonitor = createRequestMonitor({
+  snapshotFile: pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'ops-metrics.jsonl'),
+  diskPaths: [{ name: 'app-data', path: dirname(process.env.DB_PATH || './data/ytplayer.db') }, { name: 'media', path: process.env.MEDIA_DIR || './data/media' }],
+  adapters: {
+    ytdlp: () => ytdlpPool ? ytdlpPool.stats() : null,
+    media: () => media?.stats ? media.stats() : null,
+    transcription: () => notesDb.transcriptionStats?.() || null,
+    cache: () => ({ ...searchCacheCounters, memoryEntries: searchCache.size }),
+  },
+});
+app.use('*', requestMonitor.middleware());
 registerCatalogRoutes(app, { resolveListener: recommendationListener });
 
 // ============================================================================
@@ -383,6 +395,7 @@ const SEARCH_CACHE_TTL_MS = 10 * 60_000;          // fresh: answered from memory
 const SEARCH_CACHE_STALE_MS = 15 * 24 * 3600_000; // persistent copy: stale-while-revalidate for 15 days
 const searchCache = new Map(); // lowercased q -> { yt, fetchedAt }  (YouTube cards only)
 const inflightSearches = new Map(); // lowercased q -> Promise<{ yt, youtubeError? }>
+const searchCacheCounters = { hits: 0, misses: 0 };
 
 // The YouTube side of one search (up to 200 cards), shared by every caller asking
 // the same thing at the same time, and written to memory + the persistent cache.
@@ -452,11 +465,13 @@ app.get('/api/search', async (c) => {
       if (row) { hit = { yt: row.results, fetchedAt: row.fetchedAt }; searchCache.set(key, hit); }
     }
     const age = hit ? Date.now() - hit.fetchedAt : Infinity;
-    if (hit && age < SEARCH_CACHE_TTL_MS) return answer(hit.yt, { fetchedAt: hit.fetchedAt });
+    if (hit && age < SEARCH_CACHE_TTL_MS) { searchCacheCounters.hits++; return answer(hit.yt, { fetchedAt: hit.fetchedAt }); }
     if (hit && age < SEARCH_CACHE_STALE_MS) {
+      searchCacheCounters.hits++;
       fetchYoutube(q).catch(() => { /* keep serving the stale copy */ });
       return answer(hit.yt, { stale: true, fetchedAt: hit.fetchedAt });
     }
+    searchCacheCounters.misses++;
   }
   try {
     const { yt, fetchedAt } = await fetchYoutube(q);
@@ -2340,6 +2355,7 @@ const uploads = registerUploadRoutes(app, {
   parseLrc,
   sanitizeLyrics,
 });
+registerRequestMonitorRoutes(app, { adminAuth: notes.requireAdminOrToken, monitor: requestMonitor });
 const isUpload = (id) => uploads.isUploadId(id);
 
 // ============================================================================
diff --git a/server/ytdlp-pool.js b/server/ytdlp-pool.js
index 995258e..bc98030 100644
--- a/server/ytdlp-pool.js
+++ b/server/ytdlp-pool.js
@@ -108,5 +108,5 @@ export function createYtdlpPool({ ytdlpPath = '', size = 2, timeoutMs = 60_000,
     for (const w of workers) if (!w.busy) retire(w);
   }
 
-  return { run, close, recycle, get disabled() { return disabled; } };
+  return { run, close, recycle, stats: () => ({ queued: queue.length, workers: workers.length, busy: workers.filter(w => !!w.busy).length, disabled }), get disabled() { return disabled; } };
 }