diff --git a/frontend/admin.html b/frontend/admin.html index 256d2f9..291115c 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -557,6 +557,16 @@ +
+

Cache health

+

Review server copies by state and recency. Pinned copies stay out of automatic eviction.

+
+

Server media

+

+
50 rows per page
+
+
+

Library

@@ -749,6 +759,7 @@ editor: '', reports: '', quality: '', + cache: '', library: '', peers: '', history: '', @@ -757,7 +768,7 @@ audit: '', access: '', }; - const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -773,6 +784,7 @@ syncBars(); if (name === 'reports') loadReports(); if (name === 'quality') loadLyricsQuality(); + if (name === 'cache') loadCacheHealth(); if (name === 'history') loadRevs(); if (name === 'peers') loadP2p(); if (name === 'library') loadUploads(); @@ -993,6 +1005,45 @@ loadLyricsQuality(); } catch (error) { $('qualityStatus').className = 'msg err'; $('qualityStatus').textContent = error.message; } }); + + /* ---------------- cache health and pinning ---------------- */ + let cacheOffset = 0; + async function loadCacheHealth() { + $('cacheTable').innerHTML = 'Loading cache…'; + try { + const j = await api(`/api/admin/cache/health?limit=50&offset=${cacheOffset}`); + if (!j.ok) throw Error(j.error || 'Cache health unavailable.'); + const ready = j.statuses.find(x => x.status === 'ready') || {}, pending = j.statuses.find(x => ['queued', 'downloading', 'validating'].includes(x.status)), failed = j.statuses.find(x => x.status === 'failed') || {}; + $('cacheMetrics').innerHTML = [ + ['Cache used', `${bytes(j.bytes)} / ${bytes(j.budget)}`, `${Math.round(j.utilization * 100)}% of configured budget`], + ['Ready', Number(ready.count || 0).toLocaleString(), bytes(ready.bytes || 0)], + ['Pending / failed', `${Number(pending?.count || 0)} / ${Number(failed.count || 0)}`, 'Jobs in progress or needing retry'], + ['Pinned', `${bytes(j.pinnedBytes)} / ${bytes(j.pinnedBudget)}`, j.pinnedBytes > j.pinnedBudget ? 'Over cap: unpin copies to restore automatic eviction' : 'Protected from automatic eviction'], + ].map(([label, value, detail]) => `
${esc(label)}${esc(value)}${esc(detail)}
`).join(''); + $('cacheCount').textContent = `${j.total.toLocaleString()} copies · ${j.media.length ? cacheOffset + 1 : 0}–${cacheOffset + j.media.length}`; + $('cachePrev').disabled = cacheOffset === 0; $('cacheNext').disabled = cacheOffset + j.media.length >= j.total; + $('cacheTable').innerHTML = j.media.length ? `VideoStatusSizeLast accessActions${j.media.map(item => `
Generation ${item.gen}${esc(item.status)}${item.pinned ? ' · pinned' : ''}${bytes(item.bytes)}${when(Math.floor(item.lastAccess / 1000))}
${item.status === 'ready' ? `` : ''}
`).join('')}` : 'No media copies are recorded yet.'; + } catch (error) { $('cacheStatus').className = 'msg err'; $('cacheStatus').textContent = error.message; } + } + $('cacheRefresh').addEventListener('click', loadCacheHealth); + $('cachePrev').addEventListener('click', () => { cacheOffset = Math.max(0, cacheOffset - 50); loadCacheHealth(); }); + $('cacheNext').addEventListener('click', () => { cacheOffset += 50; loadCacheHealth(); }); + $('cacheTable').addEventListener('click', async event => { + const button = event.target.closest('button'); if (!button) return; + if (button.dataset.openCache) { $('songId').value = button.dataset.openCache; showTab('editor'); return; } + let path, body = {}; + if (button.dataset.cachePin) { path = `/api/admin/cache/${encodeURIComponent(button.dataset.cachePin)}/pin`; body = { pinned: button.dataset.pinned !== 'true' }; } + else if (button.dataset.cacheEvict) { + if (!confirm('Evict this server copy now? Clients may need to stream or save it again.')) return; + path = `/api/admin/cache/${encodeURIComponent(button.dataset.cacheEvict)}/evict`; + } else if (button.dataset.cacheRedownload) { + if (!confirm('Re-download this media from its source? This replaces the server copy.')) return; + path = `/api/admin/media/${encodeURIComponent(button.dataset.cacheRedownload)}/redownload`; + } else return; + $('cacheStatus').className = 'msg'; $('cacheStatus').textContent = 'Applying change…'; + try { const result = await api(path, { method: 'POST', body }); if (!result.ok) throw Error(result.error || 'Action failed.'); $('cacheStatus').className = 'msg ok'; $('cacheStatus').textContent = 'Cache updated.'; await loadCacheHealth(); } + catch (error) { $('cacheStatus').className = 'msg err'; $('cacheStatus').textContent = error.message; } + }); $('tokenForm').addEventListener('submit', async (e) => { e.preventDefault(); const label = $('tokenLabel').value.trim(); diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js index 9f1a999..0ea8003 100644 --- a/playwright.admin-audit.config.js +++ b/playwright.admin-audit.config.js @@ -1,3 +1,3 @@ const { defineConfig } = require('@playwright/test'); const classic = require('./playwright.classic.config'); -module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality)\.spec\.js/ }); +module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache)\.spec\.js/ }); diff --git a/server/admin-tools.js b/server/admin-tools.js index 72e2a5d..f44fb9e 100644 --- a/server/admin-tools.js +++ b/server/admin-tools.js @@ -51,4 +51,50 @@ export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, bac return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' }); }); + app.get('/api/admin/cache/health', adminAuth, async c => { + const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50)); + const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); + const rows = await sql.execute({ sql: "SELECT video_id,status,size,last_access,updated_at,gen,pinned FROM media_cache ORDER BY CASE WHEN status='ready' THEN 0 ELSE 1 END,last_access ASC LIMIT ? OFFSET ?", args: [limit, offset] }); + const total = await sql.execute('SELECT COUNT(*) AS total FROM media_cache'); + const grouped = await sql.execute('SELECT status,COUNT(*) AS count,COALESCE(SUM(size),0) AS bytes,COALESCE(SUM(CASE WHEN pinned=1 THEN size ELSE 0 END),0) AS pinnedBytes FROM media_cache GROUP BY status'); + const bytes = grouped.rows.reduce((sum, row) => sum + number(row.bytes), 0); + const pinnedBytes = grouped.rows.reduce((sum, row) => sum + number(row.pinnedBytes), 0); + const budget = Math.max(0, Number(process.env.MEDIA_CACHE_MAX_BYTES) || 10 * 1024 ** 3); + const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3); + return c.json({ ok: true, budget, pinnedBudget, bytes, pinnedBytes, utilization: budget ? bytes / budget : 0, + statuses: grouped.rows.map(row => ({ status: row.status, count: number(row.count), bytes: number(row.bytes) })), + total: number(total.rows[0]?.total), limit, offset, + media: rows.rows.map(row => ({ id: row.video_id, status: row.status, bytes: number(row.size), lastAccess: number(row.last_access), updatedAt: number(row.updated_at), gen: number(row.gen), pinned: !!row.pinned })) }, 200, { 'Cache-Control': 'no-store' }); + }); + + app.post('/api/admin/cache/:id/evict', adminAuth, async c => { + const id = c.req.param('id'); + if (!/^[A-Za-z0-9_-]{6,64}$/.test(id)) return c.json({ ok: false, error: 'invalid media id' }, 400); + const row = await db.getMedia(id); + if (!row) return c.json({ ok: false, error: 'media not found' }, 404); + if (row.pinned) return c.json({ ok: false, error: 'Unpin this copy before evicting it.' }, 409); + if (!media?.evictNow) return c.json({ ok: false, error: 'media cache unavailable' }, 503); + await media.evictNow(id); + await audit(db, 'admin', 'cache.evict', id, { bytes: number(row.size) }); + return c.json({ ok: true }); + }); + + app.post('/api/admin/cache/:id/pin', adminAuth, async c => { + const id = c.req.param('id'); + let body = {}; try { body = await c.req.json(); } catch {} + if (body.pinned !== true && body.pinned !== false) return c.json({ ok: false, error: 'pinned must be boolean' }, 400); + const target = await sql.execute({ sql: "SELECT size,pinned,status FROM media_cache WHERE video_id=?", args: [id] }); + const row = target.rows[0]; + if (!row) return c.json({ ok: false, error: 'media not found' }, 404); + if (body.pinned && row.status !== 'ready') return c.json({ ok: false, error: 'Only ready cache copies can be pinned.' }, 409); + const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3); + if (body.pinned && !row.pinned) { + const totals = await sql.execute('SELECT COALESCE(SUM(size),0) AS bytes FROM media_cache WHERE pinned=1'); + if (number(totals.rows[0]?.bytes) + number(row.size) > pinnedBudget) return c.json({ ok: false, error: 'Pinned cache byte cap exceeded.' }, 409); + } + await sql.execute({ sql: 'UPDATE media_cache SET pinned=? WHERE video_id=?', args: [body.pinned ? 1 : 0, id] }); + await audit(db, 'admin', body.pinned ? 'cache.pin' : 'cache.unpin', id, { bytes: number(row.size) }); + return c.json({ ok: true, pinned: body.pinned }); + }); + } diff --git a/server/admin-tools.test.js b/server/admin-tools.test.js index 3ae7189..af6739a 100644 --- a/server/admin-tools.test.js +++ b/server/admin-tools.test.js @@ -6,6 +6,7 @@ import { Hono } from 'hono'; const root = mkdtempSync(join(tmpdir(), 'ytp-admin-tools-')); process.env.DB_PATH = join(root, 'admin.db'); +process.env.MEDIA_CACHE_PINNED_MAX_BYTES = '1000'; const db = await import('./db.js'); const { registerAdminToolsRoutes } = await import('./admin-tools.js'); const app = new Hono(); @@ -17,6 +18,7 @@ beforeAll(async () => { await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] }); await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] }); await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] }); + await db.db.execute({ sql: "INSERT INTO media_cache(video_id,status,size,gen) VALUES('CacheItem001','ready',700,1),('CacheItem002','ready',400,1)" }); }); afterAll(() => rmSync(root, { recursive: true, force: true })); @@ -34,4 +36,16 @@ describe('lyrics quality queue', () => { const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json(); expect(page.songs).toHaveLength(1); }); + + test('pinning is authorized and enforces the configured pinned-byte cap', async () => { + const headers = { 'x-admin': 'yes', 'content-type': 'application/json' }; + expect((await app.request('/api/admin/cache/CacheItem001/pin', { method: 'POST', headers, body: JSON.stringify({ pinned: true }) })).status).toBe(200); + const over = await app.request('/api/admin/cache/CacheItem002/pin', { method: 'POST', headers, body: JSON.stringify({ pinned: true }) }); + expect(over.status).toBe(409); + expect((await over.json()).error).toContain('byte cap'); + const refused = await app.request('/api/admin/cache/CacheItem001/evict', { method: 'POST', headers, body: '{}' }); + expect(refused.status).toBe(409); + const health = await (await app.request('/api/admin/cache/health', { headers: { 'x-admin': 'yes' } })).json(); + expect(health.pinnedBytes).toBe(700); + }); }); diff --git a/server/db.js b/server/db.js index 900001a..491973d 100644 --- a/server/db.js +++ b/server/db.js @@ -106,7 +106,8 @@ export async function initDb() { created_at INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL DEFAULT 0, last_access INTEGER NOT NULL DEFAULT 0, - hits INTEGER NOT NULL DEFAULT 0 + hits INTEGER NOT NULL DEFAULT 0, + pinned INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_media_lru @@ -278,6 +279,7 @@ export async function initDb() { CREATE INDEX IF NOT EXISTS idx_audit_actor_action ON audit_log (actor, action, ts DESC); `); await migrateVideoMeta(); + try { await db.execute('ALTER TABLE media_cache ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0'); } catch { /* already migrated */ } // Uploads shared from devices (PWA share target): who sent it, and whether it // is listed in everyone's search (admin uploads) or reachable only by link. for (const col of ['owner TEXT', 'listed INTEGER NOT NULL DEFAULT 1']) { @@ -581,6 +583,7 @@ const MEDIA_COLS = new Set([ 'status', 'gen', 'size', 'height', 'vcodec', 'acodec', 'duration', 'optimized', 'meta', 'priority', 'auto', 'attempts', 'error', 'retry_at', 'created_at', 'updated_at', 'last_access', 'hits', 'sha256', + 'pinned', ]); function rowToObj(res, row) { @@ -618,7 +621,7 @@ export async function listMedia() { // Ready rows, least-recently-played first — the eviction order. export async function listMediaLru() { const r = await db.execute( - "SELECT video_id, size, last_access FROM media_cache WHERE status = 'ready' ORDER BY last_access ASC", + "SELECT video_id, size, last_access, pinned FROM media_cache WHERE status = 'ready' ORDER BY last_access ASC", ); return r.rows.map((row) => rowToObj(r, row)); } diff --git a/server/media-cache.js b/server/media-cache.js index 14bc7cb..f92aafe 100644 --- a/server/media-cache.js +++ b/server/media-cache.js @@ -242,7 +242,7 @@ export function createMediaCache({ const order = db.listMediaEvictionOrder ? await db.listMediaEvictionOrder() : await db.listMediaLru(); for (const r of order) { if (bytes + est <= maxBytes) break; - if (r.video_id === id || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue; + if (r.video_id === id || r.pinned || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue; await evict(r.video_id, 'budget'); bytes -= r.size || 0; } @@ -735,5 +735,6 @@ export function createMediaCache({ return n; } - return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available }; + return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available, + evictNow: id => evict(id, 'admin') }; } diff --git a/server/media-cache.test.js b/server/media-cache.test.js index a422c06..7481436 100644 --- a/server/media-cache.test.js +++ b/server/media-cache.test.js @@ -271,6 +271,21 @@ describe('media cache jobs', () => { expect(await cache.getReady('lruAAAAAAA3')).not.toBeNull(); }); + test('pinned copies are skipped while automatic LRU eviction selects another row', async () => { + await clearDb(); + const size = readFileSync(fx('good.mp4')).length; + const { cache } = makeCache({ maxBytes: size * 2 + 60 * 250 * 1024 + 50_000, duration: 10 }); + await cache.init(); + await cache.ensureCached('pinAAAAAA01', { priority: HIGH }); + await cache.ensureCached('pinAAAAAA02', { priority: HIGH }); + await dbmod.upsertMedia('pinAAAAAA01', { last_access: 1000, pinned: 1 }); + await dbmod.upsertMedia('pinAAAAAA02', { last_access: 2000 }); + await cache.ensureCached('pinAAAAAA03', { priority: HIGH }); + expect((await dbmod.getMedia('pinAAAAAA01')).pinned).toBe(1); + expect(await dbmod.getMedia('pinAAAAAA02')).toBeNull(); + expect(await cache.getReady('pinAAAAAA03')).not.toBeNull(); + }); + test('eviction follows listMediaEvictionOrder when the db provides it', async () => { await clearDb(); const size = readFileSync(fx('good.mp4')).length; diff --git a/server/p2p-db.js b/server/p2p-db.js index 5f73d23..962630b 100644 --- a/server/p2p-db.js +++ b/server/p2p-db.js @@ -243,7 +243,7 @@ export async function p2pStats() { // only then the qualifying ones, least recently played first. export async function listMediaEvictionOrder({ now, keepMinViews, keepDays, keepRecentDays }) { const r = await db.execute({ - sql: `SELECT m.video_id, m.size, m.last_access, + sql: `SELECT m.video_id, m.size, m.last_access, m.pinned, COALESCE((SELECT SUM(v.n) FROM video_views v WHERE v.video_id = m.video_id AND v.day >= ?), 0) AS views FROM media_cache m WHERE m.status = 'ready'`, diff --git a/tests/admin-cache.spec.js b/tests/admin-cache.spec.js new file mode 100644 index 0000000..bfe7f84 --- /dev/null +++ b/tests/admin-cache.spec.js @@ -0,0 +1,24 @@ +const { test, expect } = require('@playwright/test'); + +test('cache actions pin, unpin, confirm eviction and request a source re-download', async ({ page }) => { + const state = { pinned: false, evicted: false, redownloaded: false }; + await page.route('**/api/**', async route => { + const req = route.request(), url = new URL(req.url()); let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true }; + else if (url.pathname === '/api/admin/cache/health') body = { ok: true, total: 1, offset: 0, budget: 1000, pinnedBudget: 900, bytes: 700, pinnedBytes: state.pinned ? 700 : 0, utilization: .7, statuses: [{ status: 'ready', count: 1, bytes: 700 }], media: [{ id: 'CacheItem001', status: 'ready', bytes: 700, lastAccess: 1791900000000, gen: 4, pinned: state.pinned }] }; + else if (url.pathname === '/api/admin/cache/CacheItem001/pin' && req.method() === 'POST') { state.pinned = req.postDataJSON().pinned; body.pinned = state.pinned; } + else if (url.pathname === '/api/admin/cache/CacheItem001/evict' && req.method() === 'POST') state.evicted = true; + else if (url.pathname === '/api/admin/media/CacheItem001/redownload' && req.method() === 'POST') state.redownloaded = true; + else if (url.pathname === '/api/admin/media') body.media = []; + else if (url.pathname === '/api/uploads') body.uploads = []; + else if (url.pathname === '/api/admin/flags') body.flags = []; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.goto('/admin.html#cache'); + await expect(page.locator('#cacheTable')).toContainText('CacheItem001'); + await page.locator('[data-cache-pin]').click(); await expect(page.locator('[data-cache-pin]')).toHaveText('Unpin'); + await page.locator('[data-cache-pin]').click(); await expect(page.locator('[data-cache-pin]')).toHaveText('Pin'); + page.once('dialog', dialog => dialog.accept()); await page.locator('[data-cache-evict]').click(); + page.once('dialog', dialog => dialog.accept()); await page.locator('[data-cache-redownload]').click(); + expect(state).toEqual({ pinned: false, evicted: true, redownloaded: true }); +});