Deep search
Build the local video index from YouTube searches. Runs are owned by the server and continue if this page closes.
@@ -841,11 +851,12 @@
history: '',
backups: '',
analytics: '',
+ ops: '',
'deep-search': '',
audit: '',
access: '',
};
- const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['lyric-search', 'Lyrics search'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
+ const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['lyric-search', 'Lyrics search'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['ops', 'Performance'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
let tab = 'editor';
function buildTabs() {
const html = TABS.map(([k, l]) => ``).join('');
@@ -870,6 +881,7 @@
if (name === 'library') { loadUploads(); loadBulkLibrary(); loadBulkJobs(); }
if (name === 'access') loadTokens();
if (name === 'analytics') loadAnalytics();
+ if (name === 'ops') loadOpsMetrics();
if (name === 'deep-search') loadDeepSearch();
if (name === 'audit') loadAudit();
window.scrollTo({ top: 0 });
@@ -937,6 +949,30 @@
await Promise.all([loadCollections(), loadMetadata()]);
} catch (error) { $('analyticsStatus').textContent = error.message || 'Could not load server analytics.'; }
}
+ async function loadOpsMetrics() {
+ const status = $('opsStatus'); if (!status) return;
+ status.textContent = 'Sampling service health…';
+ try {
+ const { current, history = [] } = await api('/api/admin/ops/metrics');
+ const cards = [
+ ['In flight', current.inflight], ['Memory RSS', bytes(current.rssBytes)], ['CPU', `${Number(current.cpuPercent || 0).toFixed(1)}%`],
+ ['yt-dlp pool', current.ytdlp ? `${current.ytdlp.busy}/${current.ytdlp.workers} busy · ${current.ytdlp.queued} queued` : 'Unavailable'],
+ ['Media queue', current.media ? `${current.media.queued} queued · ${current.media.active ? '1 active' : 'idle'}` : 'Unavailable'],
+ ['Search cache', current.cache ? `${current.cache.hits} hits · ${current.cache.misses} misses` : 'Unavailable'],
+ ];
+ $('opsCards').innerHTML = cards.map(([name,value]) => `${esc(name)}${esc(value)}
`).join('');
+ const rows = Object.entries(current.routes || {}).sort((a,b)=>b[1].requests-a[1].requests);
+ $('opsRoutes').innerHTML = `| Route | Requests | p50 | p95 | p99 | Errors |
|---|
` + (rows.map(([path,r])=>`${esc(path)} | ${r.requests} | ${r.p50} ms | ${r.p95} ms | ${r.p99} ms | ${r.errors} (${(r.errorRate*100).toFixed(1)}%) |
`).join('') || '| No requests recorded yet. |
');
+ $('opsVolumes').innerHTML = (current.volumes || []).map(v=>`${esc(v.name)} · ${v.unavailable?'Unavailable':`${bytes(v.freeBytes)} free of ${bytes(v.totalBytes)}`}
`).join('');
+ const points = history.slice(-24).map(s=>Math.max(0,...Object.values(s.routes||{}).map(r=>r.p95)));
+ const maxPoint = Math.max(1,...points), poly = points.map((v,i)=>`${points.length < 2 ? 0 : i*100/(points.length-1)},${36-v/maxPoint*32}`).join(' ');
+ $('opsHistory').innerHTML = `Peak route p95 · last ${points.length} snapshots${maxPoint} ms `;
+ const alerts = [];
+ for (const [path,r] of rows) if (r.p95 > 5000 || r.errorRate > .1) alerts.push(`Attention: ${esc(path)} p95 ${r.p95} ms · errors ${(r.errorRate*100).toFixed(1)}%
`);
+ $('opsAlerts').innerHTML = alerts.join(''); status.textContent = `Updated ${new Date(current.at).toLocaleTimeString()}`;
+ } catch (error) { status.textContent = error.message || 'Monitoring unavailable.'; }
+ }
+ $('opsRefresh')?.addEventListener('click', loadOpsMetrics);
async function loadCollections() {
try {
const j = await api('/api/admin/collections'); if (!j.ok) return;
diff --git a/server/request-monitor.js b/server/request-monitor.js
new file mode 100644
index 0000000..8f29023
--- /dev/null
+++ b/server/request-monitor.js
@@ -0,0 +1,85 @@
+import { appendFile, mkdir, readFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { statfs } from 'node:fs/promises';
+
+const routeKey = (path) => path
+ .replace(/\/api\/media\/[^/]+/g, '/api/media/:id')
+ .replace(/\/api\/download\/[^/]+/g, '/api/download/:id')
+ .replace(/\/api\/admin\/uploads\/[^/]+/g, '/api/admin/uploads/:id')
+ .replace(/\?.*$/, '');
+const percentile = (values, p) => values.length ? values[Math.min(values.length - 1, Math.ceil(values.length * p) - 1)] : 0;
+
+export function createRequestMonitor({ snapshotFile = null, diskPaths = [], adapters = {}, now = () => Date.now(), snapshotIntervalMs = 60_000 } = {}) {
+ const routes = new Map();
+ const snapshots = [];
+ let inflight = 0;
+ let lastCpu = process.cpuUsage();
+ let lastCpuAt = now();
+ let snapshotTimer;
+
+ async function sample() {
+ const cpuNow = process.cpuUsage();
+ const elapsed = Math.max(1, now() - lastCpuAt) * 1000;
+ const cpuPct = Math.min(100 * (cpuNow.user + cpuNow.system - lastCpu.user - lastCpu.system) / elapsed, 100);
+ lastCpu = cpuNow; lastCpuAt = now();
+ const volumes = await Promise.all(diskPaths.map(async ({ name, path }) => {
+ try { const fs = await statfs(path); return { name, freeBytes: Number(fs.bavail) * Number(fs.bsize), totalBytes: Number(fs.blocks) * Number(fs.bsize) }; }
+ catch { return { name, unavailable: true }; }
+ }));
+ const routeMetrics = Object.fromEntries([...routes].map(([key, v]) => {
+ const sorted = [...v.latencies].sort((a, b) => a - b);
+ return [key, { p50: percentile(sorted, .5), p95: percentile(sorted, .95), p99: percentile(sorted, .99), requests: v.requests, errors: v.errors, errorRate: v.requests ? v.errors / v.requests : 0 }];
+ }));
+ const [ytdlp, media, transcription, cache] = await Promise.all([
+ adapters.ytdlp?.() ?? null, adapters.media?.() ?? null, adapters.transcription?.() ?? null, adapters.cache?.() ?? null,
+ ]);
+ return {
+ at: now(), routes: routeMetrics, inflight, rssBytes: process.memoryUsage().rss, cpuPercent: cpuPct,
+ ytdlp, media, transcription, cache, volumes,
+ };
+ }
+
+ async function snapshot() {
+ const value = await sample(); snapshots.push(value);
+ while (snapshots.length && now() - snapshots[0].at > 24 * 3600_000) snapshots.shift();
+ if (snapshotFile) {
+ try { await mkdir(dirname(snapshotFile), { recursive: true }); await appendFile(snapshotFile, JSON.stringify(value) + '\n'); }
+ catch (error) { console.warn('[monitor] snapshot write failed:', error.message); }
+ }
+ return value;
+ }
+ async function history() {
+ const cutoff = now() - 24 * 3600_000;
+ const out = snapshots.filter(x => x.at >= cutoff);
+ if (snapshotFile) {
+ try {
+ const persisted = (await readFile(snapshotFile, 'utf8')).split('\n').filter(Boolean).map(line => { try { return JSON.parse(line); } catch { return null; } }).filter(x => x?.at >= cutoff);
+ return [...persisted, ...out].slice(-1440);
+ } catch { /* no file yet */ }
+ }
+ return out.slice(-1440);
+ }
+
+ function middleware() {
+ return async (c, next) => {
+ const started = now(); inflight++;
+ let status = 500;
+ try { await next(); status = c.res.status; }
+ finally {
+ inflight--;
+ const key = routeKey(c.req.path);
+ let row = routes.get(key);
+ if (!row) { row = { latencies: [], requests: 0, errors: 0 }; routes.set(key, row); }
+ row.requests++; if (status >= 500) row.errors++;
+ row.latencies.push(Math.max(0, now() - started)); if (row.latencies.length > 1000) row.latencies.shift();
+ }
+ };
+ }
+
+ if (snapshotIntervalMs > 0) { snapshotTimer = setInterval(() => snapshot().catch(() => {}), snapshotIntervalMs); snapshotTimer.unref?.(); }
+ return { middleware, sample, snapshot, history, close: () => clearInterval(snapshotTimer) };
+}
+
+export function registerRequestMonitorRoutes(app, { adminAuth, monitor }) {
+ app.get('/api/admin/ops/metrics', adminAuth, async c => c.json({ ok: true, current: await monitor.sample(), history: await monitor.history() }, 200, { 'Cache-Control': 'no-store' }));
+}
diff --git a/server/request-monitor.test.js b/server/request-monitor.test.js
new file mode 100644
index 0000000..8f9a07a
--- /dev/null
+++ b/server/request-monitor.test.js
@@ -0,0 +1,19 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createRequestMonitor } from './request-monitor.js';
+
+test('request monitor records route latency, status errors, and current process health', async () => {
+ let t = 100;
+ const monitor = createRequestMonitor({ now: () => t, snapshotIntervalMs: 0, adapters: { media: () => ({ queued: 2 }) } });
+ const middleware = monitor.middleware();
+ const ctx = (path, status) => ({ req: { path }, res: { status } });
+ await middleware(ctx('/api/media/abc/status', 200), async () => { t += 8; });
+ await middleware(ctx('/api/media/xyz/status', 503), async () => { t += 12; });
+ const sample = await monitor.sample();
+ assert.equal(sample.routes['/api/media/:id/status'].requests, 2);
+ assert.equal(sample.routes['/api/media/:id/status'].errors, 1);
+ assert.equal(sample.routes['/api/media/:id/status'].p50, 8);
+ assert.equal(sample.media.queued, 2);
+ assert.ok(sample.rssBytes > 0);
+ monitor.close();
+});
diff --git a/server/server.js b/server/server.js
index 4340f7c..963b9f1 100644
--- a/server/server.js
+++ b/server/server.js
@@ -71,6 +71,7 @@ import { createYtdlpPool } from './ytdlp-pool.js';
import { createWarmQueue } from './warm-queue.js';
import { registerDeepSearchRoutes } from './deep-search.js';
import { dirname, join as pathJoin } from 'node:path';
+import { createRequestMonitor, registerRequestMonitorRoutes } from './request-monitor.js';
// A media proxy must not die because one client's stream hit an edge case
// (see /api/play cancel()): log and keep serving instead of crash-looping.
@@ -346,6 +347,17 @@ function channelToUrl(c) {
const app = new Hono();
app.use('*', logger());
+const requestMonitor = createRequestMonitor({
+ snapshotFile: pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'ops-metrics.jsonl'),
+ diskPaths: [{ name: 'app-data', path: dirname(process.env.DB_PATH || './data/ytplayer.db') }, { name: 'media', path: process.env.MEDIA_DIR || './data/media' }],
+ adapters: {
+ ytdlp: () => ytdlpPool ? ytdlpPool.stats() : null,
+ media: () => media?.stats ? media.stats() : null,
+ transcription: () => notesDb.transcriptionStats?.() || null,
+ cache: () => ({ ...searchCacheCounters, memoryEntries: searchCache.size }),
+ },
+});
+app.use('*', requestMonitor.middleware());
registerCatalogRoutes(app, { resolveListener: recommendationListener });
// ============================================================================
@@ -383,6 +395,7 @@ const SEARCH_CACHE_TTL_MS = 10 * 60_000; // fresh: answered from memory
const SEARCH_CACHE_STALE_MS = 15 * 24 * 3600_000; // persistent copy: stale-while-revalidate for 15 days
const searchCache = new Map(); // lowercased q -> { yt, fetchedAt } (YouTube cards only)
const inflightSearches = new Map(); // lowercased q -> Promise<{ yt, youtubeError? }>
+const searchCacheCounters = { hits: 0, misses: 0 };
// The YouTube side of one search (up to 200 cards), shared by every caller asking
// the same thing at the same time, and written to memory + the persistent cache.
@@ -452,11 +465,13 @@ app.get('/api/search', async (c) => {
if (row) { hit = { yt: row.results, fetchedAt: row.fetchedAt }; searchCache.set(key, hit); }
}
const age = hit ? Date.now() - hit.fetchedAt : Infinity;
- if (hit && age < SEARCH_CACHE_TTL_MS) return answer(hit.yt, { fetchedAt: hit.fetchedAt });
+ if (hit && age < SEARCH_CACHE_TTL_MS) { searchCacheCounters.hits++; return answer(hit.yt, { fetchedAt: hit.fetchedAt }); }
if (hit && age < SEARCH_CACHE_STALE_MS) {
+ searchCacheCounters.hits++;
fetchYoutube(q).catch(() => { /* keep serving the stale copy */ });
return answer(hit.yt, { stale: true, fetchedAt: hit.fetchedAt });
}
+ searchCacheCounters.misses++;
}
try {
const { yt, fetchedAt } = await fetchYoutube(q);
@@ -2340,6 +2355,7 @@ const uploads = registerUploadRoutes(app, {
parseLrc,
sanitizeLyrics,
});
+registerRequestMonitorRoutes(app, { adminAuth: notes.requireAdminOrToken, monitor: requestMonitor });
const isUpload = (id) => uploads.isUploadId(id);
// ============================================================================
diff --git a/server/ytdlp-pool.js b/server/ytdlp-pool.js
index 995258e..bc98030 100644
--- a/server/ytdlp-pool.js
+++ b/server/ytdlp-pool.js
@@ -108,5 +108,5 @@ export function createYtdlpPool({ ytdlpPath = '', size = 2, timeoutMs = 60_000,
for (const w of workers) if (!w.busy) retire(w);
}
- return { run, close, recycle, get disabled() { return disabled; } };
+ return { run, close, recycle, stats: () => ({ queued: queue.length, workers: workers.length, busy: workers.filter(w => !!w.busy).length, disabled }), get disabled() { return disabled; } };
}