Add cache pinning and health controls

frontend/admin.html

@@ -557,6 +557,16 @@
       </div>
     </section>
 
+    <section id="p-cache" class="panel">
+      <h1 class="page">Cache health</h1>
+      <p class="lead">Review server copies by state and recency. Pinned copies stay out of automatic eviction.</p>
+      <div class="analytics-grid" id="cacheMetrics"></div>
+      <div class="card"><div class="row"><h2 style="margin:0">Server media</h2><span class="grow"></span><button class="btn sm" id="cacheRefresh" type="button">Refresh</button></div>
+        <p id="cacheStatus" class="msg" role="status" aria-live="polite"></p><div class="scroll"><table class="tbl" id="cacheTable"></table></div>
+        <div class="row"><button class="btn" id="cachePrev" type="button">Previous</button><button class="btn" id="cacheNext" type="button">Next</button><span class="grow"></span><span id="cacheCount" class="muted">50 rows per page</span></div>
+      </div>
+    </section>
+
     <!-- ================= LIBRARY ================= -->
     <section id="p-library" class="panel">
       <h1 class="page">Library</h1>
@@ -749,6 +759,7 @@
     editor: '<path d="M4 20h4L19 9a2.8 2.8 0 00-4-4L4 16v4z"/><path d="M13.5 6.5l4 4"/>',
     reports: '<path d="M5 21V4M5 4h12l-2.5 4L17 12H5"/>',
     quality: '<path d="M4 5h16M4 12h16M4 19h10"/><path d="m17 18 2 2 4-5"/>',
+    cache: '<ellipse cx="12" cy="5" rx="8" ry="3"/><path d="M4 5v14c0 1.7 3.6 3 8 3s8-1.3 8-3V5M4 12c0 1.7 3.6 3 8 3s8-1.3 8-3"/>',
     library: '<path d="M9 18V6l10-2v12"/><circle cx="6.5" cy="18" r="2.5"/><circle cx="16.5" cy="16" r="2.5"/>',
     peers: '<circle cx="6" cy="12" r="2.5"/><circle cx="18" cy="6" r="2.5"/><circle cx="18" cy="18" r="2.5"/><path d="M8.3 10.9l7.4-3.8M8.3 13.1l7.4 3.8"/>',
     history: '<path d="M3 12a9 9 0 109-9 9 9 0 00-6.4 2.6L3 8"/><path d="M3 3v5h5M12 7v5l3 2"/>',
@@ -757,7 +768,7 @@
     audit: '<path d="M5 4h14v16H5zM8 8h8M8 12h8M8 16h5"/>',
     access: '<circle cx="8" cy="15" r="4"/><path d="M11 12l9-9M16 7l3 3M14 9l2 2"/>',
   };
-  const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
+  const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
   let tab = 'editor';
   function buildTabs() {
     const html = TABS.map(([k, l]) => `<button class="tab" role="tab" data-tab="${k}" aria-selected="false"><svg viewBox="0 0 24 24">${ICONS[k]}</svg><span>${l}</span>${k === 'reports' ? '<span class="badge hidden" data-badge>0</span>' : ''}</button>`).join('');
@@ -773,6 +784,7 @@
     syncBars();
     if (name === 'reports') loadReports();
     if (name === 'quality') loadLyricsQuality();
+    if (name === 'cache') loadCacheHealth();
     if (name === 'history') loadRevs();
     if (name === 'peers') loadP2p();
     if (name === 'library') loadUploads();
@@ -993,6 +1005,45 @@
       loadLyricsQuality();
     } catch (error) { $('qualityStatus').className = 'msg err'; $('qualityStatus').textContent = error.message; }
   });
+
+  /* ---------------- cache health and pinning ---------------- */
+  let cacheOffset = 0;
+  async function loadCacheHealth() {
+    $('cacheTable').innerHTML = '<tr><td class="muted">Loading cache…</td></tr>';
+    try {
+      const j = await api(`/api/admin/cache/health?limit=50&offset=${cacheOffset}`);
+      if (!j.ok) throw Error(j.error || 'Cache health unavailable.');
+      const ready = j.statuses.find(x => x.status === 'ready') || {}, pending = j.statuses.find(x => ['queued', 'downloading', 'validating'].includes(x.status)), failed = j.statuses.find(x => x.status === 'failed') || {};
+      $('cacheMetrics').innerHTML = [
+        ['Cache used', `${bytes(j.bytes)} / ${bytes(j.budget)}`, `${Math.round(j.utilization * 100)}% of configured budget`],
+        ['Ready', Number(ready.count || 0).toLocaleString(), bytes(ready.bytes || 0)],
+        ['Pending / failed', `${Number(pending?.count || 0)} / ${Number(failed.count || 0)}`, 'Jobs in progress or needing retry'],
+        ['Pinned', `${bytes(j.pinnedBytes)} / ${bytes(j.pinnedBudget)}`, j.pinnedBytes > j.pinnedBudget ? 'Over cap: unpin copies to restore automatic eviction' : 'Protected from automatic eviction'],
+      ].map(([label, value, detail]) => `<div class="metric"><span>${esc(label)}</span><b>${esc(value)}</b><small>${esc(detail)}</small></div>`).join('');
+      $('cacheCount').textContent = `${j.total.toLocaleString()} copies · ${j.media.length ? cacheOffset + 1 : 0}–${cacheOffset + j.media.length}`;
+      $('cachePrev').disabled = cacheOffset === 0; $('cacheNext').disabled = cacheOffset + j.media.length >= j.total;
+      $('cacheTable').innerHTML = j.media.length ? `<tr><th>Video</th><th>Status</th><th>Size</th><th>Last access</th><th>Actions</th></tr>${j.media.map(item => `<tr><td><button class="btn sm" data-open-cache="${esc(item.id)}">${esc(item.id)}</button><br><small>Generation ${item.gen}</small></td><td>${esc(item.status)}${item.pinned ? ' · pinned' : ''}</td><td>${bytes(item.bytes)}</td><td>${when(Math.floor(item.lastAccess / 1000))}</td><td><div class="row">${item.status === 'ready' ? `<button class="btn sm" data-cache-pin="${esc(item.id)}" data-pinned="${item.pinned}">${item.pinned ? 'Unpin' : 'Pin'}</button><button class="btn sm dng" data-cache-evict="${esc(item.id)}" ${item.pinned ? 'disabled' : ''}>Evict now</button>` : ''}<button class="btn sm" data-cache-redownload="${esc(item.id)}">Re-download</button></div></td></tr>`).join('')}` : '<tr><td class="muted">No media copies are recorded yet.</td></tr>';
+    } catch (error) { $('cacheStatus').className = 'msg err'; $('cacheStatus').textContent = error.message; }
+  }
+  $('cacheRefresh').addEventListener('click', loadCacheHealth);
+  $('cachePrev').addEventListener('click', () => { cacheOffset = Math.max(0, cacheOffset - 50); loadCacheHealth(); });
+  $('cacheNext').addEventListener('click', () => { cacheOffset += 50; loadCacheHealth(); });
+  $('cacheTable').addEventListener('click', async event => {
+    const button = event.target.closest('button'); if (!button) return;
+    if (button.dataset.openCache) { $('songId').value = button.dataset.openCache; showTab('editor'); return; }
+    let path, body = {};
+    if (button.dataset.cachePin) { path = `/api/admin/cache/${encodeURIComponent(button.dataset.cachePin)}/pin`; body = { pinned: button.dataset.pinned !== 'true' }; }
+    else if (button.dataset.cacheEvict) {
+      if (!confirm('Evict this server copy now? Clients may need to stream or save it again.')) return;
+      path = `/api/admin/cache/${encodeURIComponent(button.dataset.cacheEvict)}/evict`;
+    } else if (button.dataset.cacheRedownload) {
+      if (!confirm('Re-download this media from its source? This replaces the server copy.')) return;
+      path = `/api/admin/media/${encodeURIComponent(button.dataset.cacheRedownload)}/redownload`;
+    } else return;
+    $('cacheStatus').className = 'msg'; $('cacheStatus').textContent = 'Applying change…';
+    try { const result = await api(path, { method: 'POST', body }); if (!result.ok) throw Error(result.error || 'Action failed.'); $('cacheStatus').className = 'msg ok'; $('cacheStatus').textContent = 'Cache updated.'; await loadCacheHealth(); }
+    catch (error) { $('cacheStatus').className = 'msg err'; $('cacheStatus').textContent = error.message; }
+  });
   $('tokenForm').addEventListener('submit', async (e) => {
     e.preventDefault();
     const label = $('tokenLabel').value.trim();

playwright.admin-audit.config.js

@@ -1,3 +1,3 @@
 const { defineConfig } = require('@playwright/test');
 const classic = require('./playwright.classic.config');
-module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality)\.spec\.js/ });
+module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache)\.spec\.js/ });

server/admin-tools.js

@@ -51,4 +51,50 @@ export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, bac
     return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' });
   });
 
+  app.get('/api/admin/cache/health', adminAuth, async c => {
+    const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50));
+    const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
+    const rows = await sql.execute({ sql: "SELECT video_id,status,size,last_access,updated_at,gen,pinned FROM media_cache ORDER BY CASE WHEN status='ready' THEN 0 ELSE 1 END,last_access ASC LIMIT ? OFFSET ?", args: [limit, offset] });
+    const total = await sql.execute('SELECT COUNT(*) AS total FROM media_cache');
+    const grouped = await sql.execute('SELECT status,COUNT(*) AS count,COALESCE(SUM(size),0) AS bytes,COALESCE(SUM(CASE WHEN pinned=1 THEN size ELSE 0 END),0) AS pinnedBytes FROM media_cache GROUP BY status');
+    const bytes = grouped.rows.reduce((sum, row) => sum + number(row.bytes), 0);
+    const pinnedBytes = grouped.rows.reduce((sum, row) => sum + number(row.pinnedBytes), 0);
+    const budget = Math.max(0, Number(process.env.MEDIA_CACHE_MAX_BYTES) || 10 * 1024 ** 3);
+    const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3);
+    return c.json({ ok: true, budget, pinnedBudget, bytes, pinnedBytes, utilization: budget ? bytes / budget : 0,
+      statuses: grouped.rows.map(row => ({ status: row.status, count: number(row.count), bytes: number(row.bytes) })),
+      total: number(total.rows[0]?.total), limit, offset,
+      media: rows.rows.map(row => ({ id: row.video_id, status: row.status, bytes: number(row.size), lastAccess: number(row.last_access), updatedAt: number(row.updated_at), gen: number(row.gen), pinned: !!row.pinned })) }, 200, { 'Cache-Control': 'no-store' });
+  });
+
+  app.post('/api/admin/cache/:id/evict', adminAuth, async c => {
+    const id = c.req.param('id');
+    if (!/^[A-Za-z0-9_-]{6,64}$/.test(id)) return c.json({ ok: false, error: 'invalid media id' }, 400);
+    const row = await db.getMedia(id);
+    if (!row) return c.json({ ok: false, error: 'media not found' }, 404);
+    if (row.pinned) return c.json({ ok: false, error: 'Unpin this copy before evicting it.' }, 409);
+    if (!media?.evictNow) return c.json({ ok: false, error: 'media cache unavailable' }, 503);
+    await media.evictNow(id);
+    await audit(db, 'admin', 'cache.evict', id, { bytes: number(row.size) });
+    return c.json({ ok: true });
+  });
+
+  app.post('/api/admin/cache/:id/pin', adminAuth, async c => {
+    const id = c.req.param('id');
+    let body = {}; try { body = await c.req.json(); } catch {}
+    if (body.pinned !== true && body.pinned !== false) return c.json({ ok: false, error: 'pinned must be boolean' }, 400);
+    const target = await sql.execute({ sql: "SELECT size,pinned,status FROM media_cache WHERE video_id=?", args: [id] });
+    const row = target.rows[0];
+    if (!row) return c.json({ ok: false, error: 'media not found' }, 404);
+    if (body.pinned && row.status !== 'ready') return c.json({ ok: false, error: 'Only ready cache copies can be pinned.' }, 409);
+    const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3);
+    if (body.pinned && !row.pinned) {
+      const totals = await sql.execute('SELECT COALESCE(SUM(size),0) AS bytes FROM media_cache WHERE pinned=1');
+      if (number(totals.rows[0]?.bytes) + number(row.size) > pinnedBudget) return c.json({ ok: false, error: 'Pinned cache byte cap exceeded.' }, 409);
+    }
+    await sql.execute({ sql: 'UPDATE media_cache SET pinned=? WHERE video_id=?', args: [body.pinned ? 1 : 0, id] });
+    await audit(db, 'admin', body.pinned ? 'cache.pin' : 'cache.unpin', id, { bytes: number(row.size) });
+    return c.json({ ok: true, pinned: body.pinned });
+  });
+
 }

server/admin-tools.test.js

@@ -6,6 +6,7 @@ import { Hono } from 'hono';
 
 const root = mkdtempSync(join(tmpdir(), 'ytp-admin-tools-'));
 process.env.DB_PATH = join(root, 'admin.db');
+process.env.MEDIA_CACHE_PINNED_MAX_BYTES = '1000';
 const db = await import('./db.js');
 const { registerAdminToolsRoutes } = await import('./admin-tools.js');
 const app = new Hono();
@@ -17,6 +18,7 @@ beforeAll(async () => {
   await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] });
   await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] });
   await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] });
+  await db.db.execute({ sql: "INSERT INTO media_cache(video_id,status,size,gen) VALUES('CacheItem001','ready',700,1),('CacheItem002','ready',400,1)" });
 });
 afterAll(() => rmSync(root, { recursive: true, force: true }));
 
@@ -34,4 +36,16 @@ describe('lyrics quality queue', () => {
     const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json();
     expect(page.songs).toHaveLength(1);
   });
+
+  test('pinning is authorized and enforces the configured pinned-byte cap', async () => {
+    const headers = { 'x-admin': 'yes', 'content-type': 'application/json' };
+    expect((await app.request('/api/admin/cache/CacheItem001/pin', { method: 'POST', headers, body: JSON.stringify({ pinned: true }) })).status).toBe(200);
+    const over = await app.request('/api/admin/cache/CacheItem002/pin', { method: 'POST', headers, body: JSON.stringify({ pinned: true }) });
+    expect(over.status).toBe(409);
+    expect((await over.json()).error).toContain('byte cap');
+    const refused = await app.request('/api/admin/cache/CacheItem001/evict', { method: 'POST', headers, body: '{}' });
+    expect(refused.status).toBe(409);
+    const health = await (await app.request('/api/admin/cache/health', { headers: { 'x-admin': 'yes' } })).json();
+    expect(health.pinnedBytes).toBe(700);
+  });
 });

server/db.js

@@ -106,7 +106,8 @@ export async function initDb() {
       created_at  INTEGER NOT NULL DEFAULT 0,
       updated_at  INTEGER NOT NULL DEFAULT 0,
       last_access INTEGER NOT NULL DEFAULT 0,
-      hits        INTEGER NOT NULL DEFAULT 0
+      hits        INTEGER NOT NULL DEFAULT 0,
+      pinned      INTEGER NOT NULL DEFAULT 0
     );
 
     CREATE INDEX IF NOT EXISTS idx_media_lru
@@ -278,6 +279,7 @@ export async function initDb() {
     CREATE INDEX IF NOT EXISTS idx_audit_actor_action ON audit_log (actor, action, ts DESC);
   `);
   await migrateVideoMeta();
+  try { await db.execute('ALTER TABLE media_cache ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0'); } catch { /* already migrated */ }
   // Uploads shared from devices (PWA share target): who sent it, and whether it
   // is listed in everyone's search (admin uploads) or reachable only by link.
   for (const col of ['owner TEXT', 'listed INTEGER NOT NULL DEFAULT 1']) {
@@ -581,6 +583,7 @@ const MEDIA_COLS = new Set([
   'status', 'gen', 'size', 'height', 'vcodec', 'acodec', 'duration', 'optimized',
   'meta', 'priority', 'auto', 'attempts', 'error', 'retry_at', 'created_at',
   'updated_at', 'last_access', 'hits', 'sha256',
+  'pinned',
 ]);
 
 function rowToObj(res, row) {
@@ -618,7 +621,7 @@ export async function listMedia() {
 // Ready rows, least-recently-played first — the eviction order.
 export async function listMediaLru() {
   const r = await db.execute(
-    "SELECT video_id, size, last_access FROM media_cache WHERE status = 'ready' ORDER BY last_access ASC",
+    "SELECT video_id, size, last_access, pinned FROM media_cache WHERE status = 'ready' ORDER BY last_access ASC",
   );
   return r.rows.map((row) => rowToObj(r, row));
 }

server/media-cache.js

@@ -242,7 +242,7 @@ export function createMediaCache({
       const order = db.listMediaEvictionOrder ? await db.listMediaEvictionOrder() : await db.listMediaLru();
       for (const r of order) {
         if (bytes + est <= maxBytes) break;
-        if (r.video_id === id || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue;
+        if (r.video_id === id || r.pinned || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue;
         await evict(r.video_id, 'budget');
         bytes -= r.size || 0;
       }
@@ -735,5 +735,6 @@ export function createMediaCache({
     return n;
   }
 
-  return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available };
+  return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available,
+    evictNow: id => evict(id, 'admin') };
 }

server/media-cache.test.js

@@ -271,6 +271,21 @@ describe('media cache jobs', () => {
     expect(await cache.getReady('lruAAAAAAA3')).not.toBeNull();
   });
 
+  test('pinned copies are skipped while automatic LRU eviction selects another row', async () => {
+    await clearDb();
+    const size = readFileSync(fx('good.mp4')).length;
+    const { cache } = makeCache({ maxBytes: size * 2 + 60 * 250 * 1024 + 50_000, duration: 10 });
+    await cache.init();
+    await cache.ensureCached('pinAAAAAA01', { priority: HIGH });
+    await cache.ensureCached('pinAAAAAA02', { priority: HIGH });
+    await dbmod.upsertMedia('pinAAAAAA01', { last_access: 1000, pinned: 1 });
+    await dbmod.upsertMedia('pinAAAAAA02', { last_access: 2000 });
+    await cache.ensureCached('pinAAAAAA03', { priority: HIGH });
+    expect((await dbmod.getMedia('pinAAAAAA01')).pinned).toBe(1);
+    expect(await dbmod.getMedia('pinAAAAAA02')).toBeNull();
+    expect(await cache.getReady('pinAAAAAA03')).not.toBeNull();
+  });
+
   test('eviction follows listMediaEvictionOrder when the db provides it', async () => {
     await clearDb();
     const size = readFileSync(fx('good.mp4')).length;

server/p2p-db.js

@@ -243,7 +243,7 @@ export async function p2pStats() {
 // only then the qualifying ones, least recently played first.
 export async function listMediaEvictionOrder({ now, keepMinViews, keepDays, keepRecentDays }) {
   const r = await db.execute({
-    sql: `SELECT m.video_id, m.size, m.last_access,
+    sql: `SELECT m.video_id, m.size, m.last_access, m.pinned,
                  COALESCE((SELECT SUM(v.n) FROM video_views v
                             WHERE v.video_id = m.video_id AND v.day >= ?), 0) AS views
             FROM media_cache m WHERE m.status = 'ready'`,

tests/admin-cache.spec.js

@@ -0,0 +1,24 @@
+const { test, expect } = require('@playwright/test');
+
+test('cache actions pin, unpin, confirm eviction and request a source re-download', async ({ page }) => {
+  const state = { pinned: false, evicted: false, redownloaded: false };
+  await page.route('**/api/**', async route => {
+    const req = route.request(), url = new URL(req.url()); let body = { ok: true };
+    if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true };
+    else if (url.pathname === '/api/admin/cache/health') body = { ok: true, total: 1, offset: 0, budget: 1000, pinnedBudget: 900, bytes: 700, pinnedBytes: state.pinned ? 700 : 0, utilization: .7, statuses: [{ status: 'ready', count: 1, bytes: 700 }], media: [{ id: 'CacheItem001', status: 'ready', bytes: 700, lastAccess: 1791900000000, gen: 4, pinned: state.pinned }] };
+    else if (url.pathname === '/api/admin/cache/CacheItem001/pin' && req.method() === 'POST') { state.pinned = req.postDataJSON().pinned; body.pinned = state.pinned; }
+    else if (url.pathname === '/api/admin/cache/CacheItem001/evict' && req.method() === 'POST') state.evicted = true;
+    else if (url.pathname === '/api/admin/media/CacheItem001/redownload' && req.method() === 'POST') state.redownloaded = true;
+    else if (url.pathname === '/api/admin/media') body.media = [];
+    else if (url.pathname === '/api/uploads') body.uploads = [];
+    else if (url.pathname === '/api/admin/flags') body.flags = [];
+    await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) });
+  });
+  await page.goto('/admin.html#cache');
+  await expect(page.locator('#cacheTable')).toContainText('CacheItem001');
+  await page.locator('[data-cache-pin]').click(); await expect(page.locator('[data-cache-pin]')).toHaveText('Unpin');
+  await page.locator('[data-cache-pin]').click(); await expect(page.locator('[data-cache-pin]')).toHaveText('Pin');
+  page.once('dialog', dialog => dialog.accept()); await page.locator('[data-cache-evict]').click();
+  page.once('dialog', dialog => dialog.accept()); await page.locator('[data-cache-redownload]').click();
+  expect(state).toEqual({ pinned: false, evicted: true, redownloaded: true });
+});