josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add cache pinning and health controls

Browse Source
This commit is contained in:
Jonathan Sykes
2026-10-14 01:59:23 +08:00
parent 180a28d3e4
commit 42c68609ec
9 changed files with 161 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
frontend/admin.html
View File

@@ -557,6 +557,16 @@
</div>
</section>
<section id="p-cache" class="panel">
<h1 class="page">Cache health</h1>
<p class="lead">Review server copies by state and recency. Pinned copies stay out of automatic eviction.</p>
<div class="analytics-grid" id="cacheMetrics"></div>
<div class="card"><div class="row"><h2 style="margin:0">Server media</h2><span class="grow"></span><button class="btn sm" id="cacheRefresh" type="button">Refresh</button></div>
<p id="cacheStatus" class="msg" role="status" aria-live="polite"></p><div class="scroll"><table class="tbl" id="cacheTable"></table></div>
<div class="row"><button class="btn" id="cachePrev" type="button">Previous</button><button class="btn" id="cacheNext" type="button">Next</button><span class="grow"></span><span id="cacheCount" class="muted">50 rows per page</span></div>
</div>
</section>
<!-- ================= LIBRARY ================= -->
<section id="p-library" class="panel">
<h1 class="page">Library</h1>
@@ -749,6 +759,7 @@
editor: '<path d="M4 20h4L19 9a2.8 2.8 0 00-4-4L4 16v4z"/><path d="M13.5 6.5l4 4"/>',
reports: '<path d="M5 21V4M5 4h12l-2.5 4L17 12H5"/>',
quality: '<path d="M4 5h16M4 12h16M4 19h10"/><path d="m17 18 2 2 4-5"/>',
cache: '<ellipse cx="12" cy="5" rx="8" ry="3"/><path d="M4 5v14c0 1.7 3.6 3 8 3s8-1.3 8-3V5M4 12c0 1.7 3.6 3 8 3s8-1.3 8-3"/>',
library: '<path d="M9 18V6l10-2v12"/><circle cx="6.5" cy="18" r="2.5"/><circle cx="16.5" cy="16" r="2.5"/>',
peers: '<circle cx="6" cy="12" r="2.5"/><circle cx="18" cy="6" r="2.5"/><circle cx="18" cy="18" r="2.5"/><path d="M8.3 10.9l7.4-3.8M8.3 13.1l7.4 3.8"/>',
history: '<path d="M3 12a9 9 0 109-9 9 9 0 00-6.4 2.6L3 8"/><path d="M3 3v5h5M12 7v5l3 2"/>',
@@ -757,7 +768,7 @@
audit: '<path d="M5 4h14v16H5zM8 8h8M8 12h8M8 16h5"/>',
access: '<circle cx="8" cy="15" r="4"/><path d="M11 12l9-9M16 7l3 3M14 9l2 2"/>',
};
const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
let tab = 'editor';
function buildTabs() {
const html = TABS.map(([k, l]) => `<button class="tab" role="tab" data-tab="${k}" aria-selected="false"><svg viewBox="0 0 24 24">${ICONS[k]}</svg><span>${l}</span>${k === 'reports' ? '<span class="badge hidden" data-badge>0</span>' : ''}</button>`).join('');
@@ -773,6 +784,7 @@
syncBars();
if (name === 'reports') loadReports();
if (name === 'quality') loadLyricsQuality();
if (name === 'cache') loadCacheHealth();
if (name === 'history') loadRevs();
if (name === 'peers') loadP2p();
if (name === 'library') loadUploads();
@@ -993,6 +1005,45 @@
loadLyricsQuality();
} catch (error) { $('qualityStatus').className = 'msg err'; $('qualityStatus').textContent = error.message; }
});
/* ---------------- cache health and pinning ---------------- */
let cacheOffset = 0;
async function loadCacheHealth() {
$('cacheTable').innerHTML = '<tr><td class="muted">Loading cache…</td></tr>';
try {
const j = await api(`/api/admin/cache/health?limit=50&offset=${cacheOffset}`);
if (!j.ok) throw Error(j.error || 'Cache health unavailable.');
const ready = j.statuses.find(x => x.status === 'ready') || {}, pending = j.statuses.find(x => ['queued', 'downloading', 'validating'].includes(x.status)), failed = j.statuses.find(x => x.status === 'failed') || {};
$('cacheMetrics').innerHTML = [
['Cache used', `${bytes(j.bytes)} / ${bytes(j.budget)}`, `${Math.round(j.utilization * 100)}% of configured budget`],
['Ready', Number(ready.count || 0).toLocaleString(), bytes(ready.bytes || 0)],
['Pending / failed', `${Number(pending?.count || 0)} / ${Number(failed.count || 0)}`, 'Jobs in progress or needing retry'],
['Pinned', `${bytes(j.pinnedBytes)} / ${bytes(j.pinnedBudget)}`, j.pinnedBytes > j.pinnedBudget ? 'Over cap: unpin copies to restore automatic eviction' : 'Protected from automatic eviction'],
].map(([label, value, detail]) => `<div class="metric"><span>${esc(label)}</span><b>${esc(value)}</b><small>${esc(detail)}</small></div>`).join('');
$('cacheCount').textContent = `${j.total.toLocaleString()} copies · ${j.media.length ? cacheOffset + 1 : 0}–${cacheOffset + j.media.length}`;
$('cachePrev').disabled = cacheOffset === 0; $('cacheNext').disabled = cacheOffset + j.media.length >= j.total;
$('cacheTable').innerHTML = j.media.length ? `<tr><th>Video</th><th>Status</th><th>Size</th><th>Last access</th><th>Actions</th></tr>${j.media.map(item => `<tr><td><button class="btn sm" data-open-cache="${esc(item.id)}">${esc(item.id)}</button><br><small>Generation ${item.gen}</small></td><td>${esc(item.status)}${item.pinned ? ' · pinned' : ''}</td><td>${bytes(item.bytes)}</td><td>${when(Math.floor(item.lastAccess / 1000))}</td><td><div class="row">${item.status === 'ready' ? `<button class="btn sm" data-cache-pin="${esc(item.id)}" data-pinned="${item.pinned}">${item.pinned ? 'Unpin' : 'Pin'}</button><button class="btn sm dng" data-cache-evict="${esc(item.id)}" ${item.pinned ? 'disabled' : ''}>Evict now</button>` : ''}<button class="btn sm" data-cache-redownload="${esc(item.id)}">Re-download</button></div></td></tr>`).join('')}` : '<tr><td class="muted">No media copies are recorded yet.</td></tr>';
} catch (error) { $('cacheStatus').className = 'msg err'; $('cacheStatus').textContent = error.message; }
}
$('cacheRefresh').addEventListener('click', loadCacheHealth);
$('cachePrev').addEventListener('click', () => { cacheOffset = Math.max(0, cacheOffset - 50); loadCacheHealth(); });
$('cacheNext').addEventListener('click', () => { cacheOffset += 50; loadCacheHealth(); });
$('cacheTable').addEventListener('click', async event => {
const button = event.target.closest('button'); if (!button) return;
if (button.dataset.openCache) { $('songId').value = button.dataset.openCache; showTab('editor'); return; }
let path, body = {};
if (button.dataset.cachePin) { path = `/api/admin/cache/${encodeURIComponent(button.dataset.cachePin)}/pin`; body = { pinned: button.dataset.pinned !== 'true' }; }
else if (button.dataset.cacheEvict) {
if (!confirm('Evict this server copy now? Clients may need to stream or save it again.')) return;
path = `/api/admin/cache/${encodeURIComponent(button.dataset.cacheEvict)}/evict`;
} else if (button.dataset.cacheRedownload) {
if (!confirm('Re-download this media from its source? This replaces the server copy.')) return;
path = `/api/admin/media/${encodeURIComponent(button.dataset.cacheRedownload)}/redownload`;
} else return;
$('cacheStatus').className = 'msg'; $('cacheStatus').textContent = 'Applying change…';
try { const result = await api(path, { method: 'POST', body }); if (!result.ok) throw Error(result.error || 'Action failed.'); $('cacheStatus').className = 'msg ok'; $('cacheStatus').textContent = 'Cache updated.'; await loadCacheHealth(); }
catch (error) { $('cacheStatus').className = 'msg err'; $('cacheStatus').textContent = error.message; }
});
$('tokenForm').addEventListener('submit', async (e) => {
e.preventDefault();
const label = $('tokenLabel').value.trim();

2
playwright.admin-audit.config.js
View File

@@ -1,3 +1,3 @@
const { defineConfig } = require('@playwright/test');
const classic = require('./playwright.classic.config');
module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality)\.spec\.js/ });
module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache)\.spec\.js/ });

46
server/admin-tools.js
View File

@@ -51,4 +51,50 @@ export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, bac
return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' });
});
app.get('/api/admin/cache/health', adminAuth, async c => {
const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50));
const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
const rows = await sql.execute({ sql: "SELECT video_id,status,size,last_access,updated_at,gen,pinned FROM media_cache ORDER BY CASE WHEN status='ready' THEN 0 ELSE 1 END,last_access ASC LIMIT ? OFFSET ?", args: [limit, offset] });
const total = await sql.execute('SELECT COUNT(*) AS total FROM media_cache');
const grouped = await sql.execute('SELECT status,COUNT(*) AS count,COALESCE(SUM(size),0) AS bytes,COALESCE(SUM(CASE WHEN pinned=1 THEN size ELSE 0 END),0) AS pinnedBytes FROM media_cache GROUP BY status');
const bytes = grouped.rows.reduce((sum, row) => sum + number(row.bytes), 0);
const pinnedBytes = grouped.rows.reduce((sum, row) => sum + number(row.pinnedBytes), 0);
const budget = Math.max(0, Number(process.env.MEDIA_CACHE_MAX_BYTES) || 10 * 1024 ** 3);
const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3);
return c.json({ ok: true, budget, pinnedBudget, bytes, pinnedBytes, utilization: budget ? bytes / budget : 0,
statuses: grouped.rows.map(row => ({ status: row.status, count: number(row.count), bytes: number(row.bytes) })),
total: number(total.rows[0]?.total), limit, offset,
media: rows.rows.map(row => ({ id: row.video_id, status: row.status, bytes: number(row.size), lastAccess: number(row.last_access), updatedAt: number(row.updated_at), gen: number(row.gen), pinned: !!row.pinned })) }, 200, { 'Cache-Control': 'no-store' });
});
app.post('/api/admin/cache/:id/evict', adminAuth, async c => {
const id = c.req.param('id');
if (!/^[A-Za-z0-9_-]{6,64}$/.test(id)) return c.json({ ok: false, error: 'invalid media id' }, 400);
const row = await db.getMedia(id);
if (!row) return c.json({ ok: false, error: 'media not found' }, 404);
if (row.pinned) return c.json({ ok: false, error: 'Unpin this copy before evicting it.' }, 409);
if (!media?.evictNow) return c.json({ ok: false, error: 'media cache unavailable' }, 503);
await media.evictNow(id);
await audit(db, 'admin', 'cache.evict', id, { bytes: number(row.size) });
return c.json({ ok: true });
});
app.post('/api/admin/cache/:id/pin', adminAuth, async c => {
const id = c.req.param('id');
let body = {}; try { body = await c.req.json(); } catch {}
if (body.pinned !== true && body.pinned !== false) return c.json({ ok: false, error: 'pinned must be boolean' }, 400);
const target = await sql.execute({ sql: "SELECT size,pinned,status FROM media_cache WHERE video_id=?", args: [id] });
const row = target.rows[0];
if (!row) return c.json({ ok: false, error: 'media not found' }, 404);
if (body.pinned && row.status !== 'ready') return c.json({ ok: false, error: 'Only ready cache copies can be pinned.' }, 409);
const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3);
if (body.pinned && !row.pinned) {
const totals = await sql.execute('SELECT COALESCE(SUM(size),0) AS bytes FROM media_cache WHERE pinned=1');
if (number(totals.rows[0]?.bytes) + number(row.size) > pinnedBudget) return c.json({ ok: false, error: 'Pinned cache byte cap exceeded.' }, 409);
}
await sql.execute({ sql: 'UPDATE media_cache SET pinned=? WHERE video_id=?', args: [body.pinned ? 1 : 0, id] });
await audit(db, 'admin', body.pinned ? 'cache.pin' : 'cache.unpin', id, { bytes: number(row.size) });
return c.json({ ok: true, pinned: body.pinned });
});
}

14
server/admin-tools.test.js
View File

@@ -6,6 +6,7 @@ import { Hono } from 'hono';
const root = mkdtempSync(join(tmpdir(), 'ytp-admin-tools-'));
process.env.DB_PATH = join(root, 'admin.db');
process.env.MEDIA_CACHE_PINNED_MAX_BYTES = '1000';
const db = await import('./db.js');
const { registerAdminToolsRoutes } = await import('./admin-tools.js');
const app = new Hono();
@@ -17,6 +18,7 @@ beforeAll(async () => {
await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] });
await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] });
await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] });
await db.db.execute({ sql: "INSERT INTO media_cache(video_id,status,size,gen) VALUES('CacheItem001','ready',700,1),('CacheItem002','ready',400,1)" });
});
afterAll(() => rmSync(root, { recursive: true, force: true }));
@@ -34,4 +36,16 @@ describe('lyrics quality queue', () => {
const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json();
expect(page.songs).toHaveLength(1);
});
test('pinning is authorized and enforces the configured pinned-byte cap', async () => {
const headers = { 'x-admin': 'yes', 'content-type': 'application/json' };
expect((await app.request('/api/admin/cache/CacheItem001/pin', { method: 'POST', headers, body: JSON.stringify({ pinned: true }) })).status).toBe(200);
const over = await app.request('/api/admin/cache/CacheItem002/pin', { method: 'POST', headers, body: JSON.stringify({ pinned: true }) });
expect(over.status).toBe(409);
expect((await over.json()).error).toContain('byte cap');
const refused = await app.request('/api/admin/cache/CacheItem001/evict', { method: 'POST', headers, body: '{}' });
expect(refused.status).toBe(409);
const health = await (await app.request('/api/admin/cache/health', { headers: { 'x-admin': 'yes' } })).json();
expect(health.pinnedBytes).toBe(700);
});
});

7
server/db.js
View File

@@ -106,7 +106,8 @@ export async function initDb() {
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
last_access INTEGER NOT NULL DEFAULT 0,
hits INTEGER NOT NULL DEFAULT 0
hits INTEGER NOT NULL DEFAULT 0,
pinned INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_media_lru
@@ -278,6 +279,7 @@ export async function initDb() {
CREATE INDEX IF NOT EXISTS idx_audit_actor_action ON audit_log (actor, action, ts DESC);
`);
await migrateVideoMeta();
try { await db.execute('ALTER TABLE media_cache ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0'); } catch { /* already migrated */ }
// Uploads shared from devices (PWA share target): who sent it, and whether it
// is listed in everyone's search (admin uploads) or reachable only by link.
for (const col of ['owner TEXT', 'listed INTEGER NOT NULL DEFAULT 1']) {
@@ -581,6 +583,7 @@ const MEDIA_COLS = new Set([
'status', 'gen', 'size', 'height', 'vcodec', 'acodec', 'duration', 'optimized',
'meta', 'priority', 'auto', 'attempts', 'error', 'retry_at', 'created_at',
'updated_at', 'last_access', 'hits', 'sha256',
'pinned',
]);
function rowToObj(res, row) {
@@ -618,7 +621,7 @@ export async function listMedia() {
// Ready rows, least-recently-played first — the eviction order.
export async function listMediaLru() {
const r = await db.execute(
"SELECT video_id, size, last_access FROM media_cache WHERE status = 'ready' ORDER BY last_access ASC",
"SELECT video_id, size, last_access, pinned FROM media_cache WHERE status = 'ready' ORDER BY last_access ASC",
);
return r.rows.map((row) => rowToObj(r, row));
}

5
server/media-cache.js
View File

@@ -242,7 +242,7 @@ export function createMediaCache({
const order = db.listMediaEvictionOrder ? await db.listMediaEvictionOrder() : await db.listMediaLru();
for (const r of order) {
if (bytes + est <= maxBytes) break;
if (r.video_id === id || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue;
if (r.video_id === id || r.pinned || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue;
await evict(r.video_id, 'budget');
bytes -= r.size || 0;
}
@@ -735,5 +735,6 @@ export function createMediaCache({
return n;
}
return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available };
return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available,
evictNow: id => evict(id, 'admin') };
}

15
server/media-cache.test.js
View File

@@ -271,6 +271,21 @@ describe('media cache jobs', () => {
expect(await cache.getReady('lruAAAAAAA3')).not.toBeNull();
});
test('pinned copies are skipped while automatic LRU eviction selects another row', async () => {
await clearDb();
const size = readFileSync(fx('good.mp4')).length;
const { cache } = makeCache({ maxBytes: size * 2 + 60 * 250 * 1024 + 50_000, duration: 10 });
await cache.init();
await cache.ensureCached('pinAAAAAA01', { priority: HIGH });
await cache.ensureCached('pinAAAAAA02', { priority: HIGH });
await dbmod.upsertMedia('pinAAAAAA01', { last_access: 1000, pinned: 1 });
await dbmod.upsertMedia('pinAAAAAA02', { last_access: 2000 });
await cache.ensureCached('pinAAAAAA03', { priority: HIGH });
expect((await dbmod.getMedia('pinAAAAAA01')).pinned).toBe(1);
expect(await dbmod.getMedia('pinAAAAAA02')).toBeNull();
expect(await cache.getReady('pinAAAAAA03')).not.toBeNull();
});
test('eviction follows listMediaEvictionOrder when the db provides it', async () => {
await clearDb();
const size = readFileSync(fx('good.mp4')).length;

2
server/p2p-db.js
View File

@@ -243,7 +243,7 @@ export async function p2pStats() {
// only then the qualifying ones, least recently played first.
export async function listMediaEvictionOrder({ now, keepMinViews, keepDays, keepRecentDays }) {
const r = await db.execute({
sql: `SELECT m.video_id, m.size, m.last_access,
sql: `SELECT m.video_id, m.size, m.last_access, m.pinned,
COALESCE((SELECT SUM(v.n) FROM video_views v
WHERE v.video_id = m.video_id AND v.day >= ?), 0) AS views
FROM media_cache m WHERE m.status = 'ready'`,

24
tests/admin-cache.spec.js Normal file
View File

@@ -0,0 +1,24 @@
const { test, expect } = require('@playwright/test');
test('cache actions pin, unpin, confirm eviction and request a source re-download', async ({ page }) => {
const state = { pinned: false, evicted: false, redownloaded: false };
await page.route('**/api/**', async route => {
const req = route.request(), url = new URL(req.url()); let body = { ok: true };
if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true };
else if (url.pathname === '/api/admin/cache/health') body = { ok: true, total: 1, offset: 0, budget: 1000, pinnedBudget: 900, bytes: 700, pinnedBytes: state.pinned ? 700 : 0, utilization: .7, statuses: [{ status: 'ready', count: 1, bytes: 700 }], media: [{ id: 'CacheItem001', status: 'ready', bytes: 700, lastAccess: 1791900000000, gen: 4, pinned: state.pinned }] };
else if (url.pathname === '/api/admin/cache/CacheItem001/pin' && req.method() === 'POST') { state.pinned = req.postDataJSON().pinned; body.pinned = state.pinned; }
else if (url.pathname === '/api/admin/cache/CacheItem001/evict' && req.method() === 'POST') state.evicted = true;
else if (url.pathname === '/api/admin/media/CacheItem001/redownload' && req.method() === 'POST') state.redownloaded = true;
else if (url.pathname === '/api/admin/media') body.media = [];
else if (url.pathname === '/api/uploads') body.uploads = [];
else if (url.pathname === '/api/admin/flags') body.flags = [];
await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) });
});
await page.goto('/admin.html#cache');
await expect(page.locator('#cacheTable')).toContainText('CacheItem001');
await page.locator('[data-cache-pin]').click(); await expect(page.locator('[data-cache-pin]')).toHaveText('Unpin');
await page.locator('[data-cache-pin]').click(); await expect(page.locator('[data-cache-pin]')).toHaveText('Pin');
page.once('dialog', dialog => dialog.accept()); await page.locator('[data-cache-evict]').click();
page.once('dialog', dialog => dialog.accept()); await page.locator('[data-cache-redownload]').click();
expect(state).toEqual({ pinned: false, evicted: true, redownloaded: true });
});