josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add cache pinning and health controls

Browse Source
This commit is contained in:
Jonathan Sykes
2026-10-14 01:59:23 +08:00
parent 180a28d3e4
commit 42c68609ec
9 changed files with 161 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
server/admin-tools.js
View File

@@ -51,4 +51,50 @@ export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, bac
return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' });
});
app.get('/api/admin/cache/health', adminAuth, async c => {
const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50));
const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
const rows = await sql.execute({ sql: "SELECT video_id,status,size,last_access,updated_at,gen,pinned FROM media_cache ORDER BY CASE WHEN status='ready' THEN 0 ELSE 1 END,last_access ASC LIMIT ? OFFSET ?", args: [limit, offset] });
const total = await sql.execute('SELECT COUNT(*) AS total FROM media_cache');
const grouped = await sql.execute('SELECT status,COUNT(*) AS count,COALESCE(SUM(size),0) AS bytes,COALESCE(SUM(CASE WHEN pinned=1 THEN size ELSE 0 END),0) AS pinnedBytes FROM media_cache GROUP BY status');
const bytes = grouped.rows.reduce((sum, row) => sum + number(row.bytes), 0);
const pinnedBytes = grouped.rows.reduce((sum, row) => sum + number(row.pinnedBytes), 0);
const budget = Math.max(0, Number(process.env.MEDIA_CACHE_MAX_BYTES) || 10 * 1024 ** 3);
const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3);
return c.json({ ok: true, budget, pinnedBudget, bytes, pinnedBytes, utilization: budget ? bytes / budget : 0,
statuses: grouped.rows.map(row => ({ status: row.status, count: number(row.count), bytes: number(row.bytes) })),
total: number(total.rows[0]?.total), limit, offset,
media: rows.rows.map(row => ({ id: row.video_id, status: row.status, bytes: number(row.size), lastAccess: number(row.last_access), updatedAt: number(row.updated_at), gen: number(row.gen), pinned: !!row.pinned })) }, 200, { 'Cache-Control': 'no-store' });
});
app.post('/api/admin/cache/:id/evict', adminAuth, async c => {
const id = c.req.param('id');
if (!/^[A-Za-z0-9_-]{6,64}$/.test(id)) return c.json({ ok: false, error: 'invalid media id' }, 400);
const row = await db.getMedia(id);
if (!row) return c.json({ ok: false, error: 'media not found' }, 404);
if (row.pinned) return c.json({ ok: false, error: 'Unpin this copy before evicting it.' }, 409);
if (!media?.evictNow) return c.json({ ok: false, error: 'media cache unavailable' }, 503);
await media.evictNow(id);
await audit(db, 'admin', 'cache.evict', id, { bytes: number(row.size) });
return c.json({ ok: true });
});
app.post('/api/admin/cache/:id/pin', adminAuth, async c => {
const id = c.req.param('id');
let body = {}; try { body = await c.req.json(); } catch {}
if (body.pinned !== true && body.pinned !== false) return c.json({ ok: false, error: 'pinned must be boolean' }, 400);
const target = await sql.execute({ sql: "SELECT size,pinned,status FROM media_cache WHERE video_id=?", args: [id] });
const row = target.rows[0];
if (!row) return c.json({ ok: false, error: 'media not found' }, 404);
if (body.pinned && row.status !== 'ready') return c.json({ ok: false, error: 'Only ready cache copies can be pinned.' }, 409);
const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3);
if (body.pinned && !row.pinned) {
const totals = await sql.execute('SELECT COALESCE(SUM(size),0) AS bytes FROM media_cache WHERE pinned=1');
if (number(totals.rows[0]?.bytes) + number(row.size) > pinnedBudget) return c.json({ ok: false, error: 'Pinned cache byte cap exceeded.' }, 409);
}
await sql.execute({ sql: 'UPDATE media_cache SET pinned=? WHERE video_id=?', args: [body.pinned ? 1 : 0, id] });
await audit(db, 'admin', body.pinned ? 'cache.pin' : 'cache.unpin', id, { bytes: number(row.size) });
return c.json({ ok: true, pinned: body.pinned });
});
}

14
server/admin-tools.test.js
View File

@@ -6,6 +6,7 @@ import { Hono } from 'hono';
const root = mkdtempSync(join(tmpdir(), 'ytp-admin-tools-'));
process.env.DB_PATH = join(root, 'admin.db');
process.env.MEDIA_CACHE_PINNED_MAX_BYTES = '1000';
const db = await import('./db.js');
const { registerAdminToolsRoutes } = await import('./admin-tools.js');
const app = new Hono();
@@ -17,6 +18,7 @@ beforeAll(async () => {
await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] });
await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] });
await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] });
await db.db.execute({ sql: "INSERT INTO media_cache(video_id,status,size,gen) VALUES('CacheItem001','ready',700,1),('CacheItem002','ready',400,1)" });
});
afterAll(() => rmSync(root, { recursive: true, force: true }));
@@ -34,4 +36,16 @@ describe('lyrics quality queue', () => {
const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json();
expect(page.songs).toHaveLength(1);
});
test('pinning is authorized and enforces the configured pinned-byte cap', async () => {
const headers = { 'x-admin': 'yes', 'content-type': 'application/json' };
expect((await app.request('/api/admin/cache/CacheItem001/pin', { method: 'POST', headers, body: JSON.stringify({ pinned: true }) })).status).toBe(200);
const over = await app.request('/api/admin/cache/CacheItem002/pin', { method: 'POST', headers, body: JSON.stringify({ pinned: true }) });
expect(over.status).toBe(409);
expect((await over.json()).error).toContain('byte cap');
const refused = await app.request('/api/admin/cache/CacheItem001/evict', { method: 'POST', headers, body: '{}' });
expect(refused.status).toBe(409);
const health = await (await app.request('/api/admin/cache/health', { headers: { 'x-admin': 'yes' } })).json();
expect(health.pinnedBytes).toBe(700);
});
});

7
server/db.js
View File

@@ -106,7 +106,8 @@ export async function initDb() {
created_at INTEGER NOT NULL DEFAULT 0,
updated_at INTEGER NOT NULL DEFAULT 0,
last_access INTEGER NOT NULL DEFAULT 0,
hits INTEGER NOT NULL DEFAULT 0
hits INTEGER NOT NULL DEFAULT 0,
pinned INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_media_lru
@@ -278,6 +279,7 @@ export async function initDb() {
CREATE INDEX IF NOT EXISTS idx_audit_actor_action ON audit_log (actor, action, ts DESC);
`);
await migrateVideoMeta();
try { await db.execute('ALTER TABLE media_cache ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0'); } catch { /* already migrated */ }
// Uploads shared from devices (PWA share target): who sent it, and whether it
// is listed in everyone's search (admin uploads) or reachable only by link.
for (const col of ['owner TEXT', 'listed INTEGER NOT NULL DEFAULT 1']) {
@@ -581,6 +583,7 @@ const MEDIA_COLS = new Set([
'status', 'gen', 'size', 'height', 'vcodec', 'acodec', 'duration', 'optimized',
'meta', 'priority', 'auto', 'attempts', 'error', 'retry_at', 'created_at',
'updated_at', 'last_access', 'hits', 'sha256',
'pinned',
]);
function rowToObj(res, row) {
@@ -618,7 +621,7 @@ export async function listMedia() {
// Ready rows, least-recently-played first — the eviction order.
export async function listMediaLru() {
const r = await db.execute(
"SELECT video_id, size, last_access FROM media_cache WHERE status = 'ready' ORDER BY last_access ASC",
"SELECT video_id, size, last_access, pinned FROM media_cache WHERE status = 'ready' ORDER BY last_access ASC",
);
return r.rows.map((row) => rowToObj(r, row));
}

5
server/media-cache.js
View File

@@ -242,7 +242,7 @@ export function createMediaCache({
const order = db.listMediaEvictionOrder ? await db.listMediaEvictionOrder() : await db.listMediaLru();
for (const r of order) {
if (bytes + est <= maxBytes) break;
if (r.video_id === id || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue;
if (r.video_id === id || r.pinned || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue;
await evict(r.video_id, 'budget');
bytes -= r.size || 0;
}
@@ -735,5 +735,6 @@ export function createMediaCache({
return n;
}
return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available };
return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available,
evictNow: id => evict(id, 'admin') };
}

15
server/media-cache.test.js
View File

@@ -271,6 +271,21 @@ describe('media cache jobs', () => {
expect(await cache.getReady('lruAAAAAAA3')).not.toBeNull();
});
test('pinned copies are skipped while automatic LRU eviction selects another row', async () => {
await clearDb();
const size = readFileSync(fx('good.mp4')).length;
const { cache } = makeCache({ maxBytes: size * 2 + 60 * 250 * 1024 + 50_000, duration: 10 });
await cache.init();
await cache.ensureCached('pinAAAAAA01', { priority: HIGH });
await cache.ensureCached('pinAAAAAA02', { priority: HIGH });
await dbmod.upsertMedia('pinAAAAAA01', { last_access: 1000, pinned: 1 });
await dbmod.upsertMedia('pinAAAAAA02', { last_access: 2000 });
await cache.ensureCached('pinAAAAAA03', { priority: HIGH });
expect((await dbmod.getMedia('pinAAAAAA01')).pinned).toBe(1);
expect(await dbmod.getMedia('pinAAAAAA02')).toBeNull();
expect(await cache.getReady('pinAAAAAA03')).not.toBeNull();
});
test('eviction follows listMediaEvictionOrder when the db provides it', async () => {
await clearDb();
const size = readFileSync(fx('good.mp4')).length;

2
server/p2p-db.js
View File

@@ -243,7 +243,7 @@ export async function p2pStats() {
// only then the qualifying ones, least recently played first.
export async function listMediaEvictionOrder({ now, keepMinViews, keepDays, keepRecentDays }) {
const r = await db.execute({
sql: `SELECT m.video_id, m.size, m.last_access,
sql: `SELECT m.video_id, m.size, m.last_access, m.pinned,
COALESCE((SELECT SUM(v.n) FROM video_views v
WHERE v.video_id = m.video_id AND v.day >= ?), 0) AS views
FROM media_cache m WHERE m.status = 'ready'`,