150 lines
11 KiB
JavaScript
150 lines
11 KiB
JavaScript
import { audit } from './audit-log.js';
|
|
|
|
const clientOf = db => typeof db?.execute === 'function' ? db : db?.db;
|
|
const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } };
|
|
const number = value => Number(value) || 0;
|
|
|
|
export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null, backfillStep = null }) {
|
|
const sql = clientOf(db);
|
|
|
|
app.get('/api/admin/lyrics/quality', adminAuth, async c => {
|
|
const minLines = Math.max(1, Math.min(200, Number(c.req.query('minLines')) || 8));
|
|
const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
|
|
const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50));
|
|
const category = ['all', 'missing', 'short', 'automatic', 'untimed', 'reports'].includes(c.req.query('category')) ? c.req.query('category') : 'all';
|
|
const conditions = {
|
|
missing: 'n.video_id IS NULL',
|
|
short: 'n.video_id IS NOT NULL AND COALESCE(json_array_length(json_extract(n.data,\'$.lines\')),0) < ?',
|
|
automatic: "n.source='auto'",
|
|
untimed: "EXISTS (SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL)",
|
|
reports: "EXISTS (SELECT 1 FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open')",
|
|
};
|
|
const parts = category === 'all'
|
|
? [conditions.missing, conditions.short, conditions.automatic, conditions.untimed, conditions.reports]
|
|
: [conditions[category]];
|
|
const where = `WHERE (${parts.join(' OR ')})`;
|
|
const args = category === 'short' ? [minLines] : category === 'all' ? [minLines] : [];
|
|
const query = `SELECT m.id,m.card,m.updated_at,n.data AS lyrics,n.source AS lyricSource,n.updated_at AS lyricUpdated,
|
|
COALESCE(json_array_length(json_extract(n.data,'$.lines')),0) AS lineCount,
|
|
EXISTS(SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL) AS untimed,
|
|
(SELECT COUNT(*) FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open') AS openReports
|
|
FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics'
|
|
${where} ORDER BY openReports DESC,lyricUpdated ASC,m.updated_at DESC LIMIT ? OFFSET ?`;
|
|
const rows = await sql.execute({ sql: query, args: [...args, limit, offset] });
|
|
const count = await sql.execute({ sql: `SELECT COUNT(*) AS total FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics' ${where}`, args });
|
|
const songs = rows.rows.map(row => {
|
|
const card = parse(row.card);
|
|
let untimed = !!row.untimed;
|
|
let lines = number(row.lineCount);
|
|
if (row.lyrics) {
|
|
const data = parse(row.lyrics); lines ||= Array.isArray(data.lines) ? data.lines.length : 0;
|
|
if (Array.isArray(data.lines)) untimed ||= data.lines.some(line => line.t == null);
|
|
}
|
|
const reasons = [];
|
|
if (!row.lyrics) reasons.push('missing');
|
|
else if (lines < minLines) reasons.push('short');
|
|
if (row.lyricSource === 'auto') reasons.push('automatic');
|
|
if (untimed) reasons.push('untimed');
|
|
if (number(row.openReports)) reasons.push('reports');
|
|
return { id: row.id, title: card.title || row.id, channel: card.channel || '', lines, source: row.lyricSource || null, untimed, openReports: number(row.openReports), reasons, updatedAt: number(row.lyricUpdated) };
|
|
});
|
|
return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' });
|
|
});
|
|
|
|
app.get('/api/admin/cache/health', adminAuth, async c => {
|
|
const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50));
|
|
const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
|
|
const rows = await sql.execute({ sql: "SELECT video_id,status,size,last_access,updated_at,gen,pinned FROM media_cache ORDER BY CASE WHEN status='ready' THEN 0 ELSE 1 END,last_access ASC LIMIT ? OFFSET ?", args: [limit, offset] });
|
|
const total = await sql.execute('SELECT COUNT(*) AS total FROM media_cache');
|
|
const grouped = await sql.execute('SELECT status,COUNT(*) AS count,COALESCE(SUM(size),0) AS bytes,COALESCE(SUM(CASE WHEN pinned=1 THEN size ELSE 0 END),0) AS pinnedBytes FROM media_cache GROUP BY status');
|
|
const bytes = grouped.rows.reduce((sum, row) => sum + number(row.bytes), 0);
|
|
const pinnedBytes = grouped.rows.reduce((sum, row) => sum + number(row.pinnedBytes), 0);
|
|
const budget = Math.max(0, Number(process.env.MEDIA_CACHE_MAX_BYTES) || 10 * 1024 ** 3);
|
|
const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3);
|
|
return c.json({ ok: true, budget, pinnedBudget, bytes, pinnedBytes, utilization: budget ? bytes / budget : 0,
|
|
statuses: grouped.rows.map(row => ({ status: row.status, count: number(row.count), bytes: number(row.bytes) })),
|
|
total: number(total.rows[0]?.total), limit, offset,
|
|
media: rows.rows.map(row => ({ id: row.video_id, status: row.status, bytes: number(row.size), lastAccess: number(row.last_access), updatedAt: number(row.updated_at), gen: number(row.gen), pinned: !!row.pinned })) }, 200, { 'Cache-Control': 'no-store' });
|
|
});
|
|
|
|
app.post('/api/admin/cache/:id/evict', adminAuth, async c => {
|
|
const id = c.req.param('id');
|
|
if (!/^[A-Za-z0-9_-]{6,64}$/.test(id)) return c.json({ ok: false, error: 'invalid media id' }, 400);
|
|
const row = await db.getMedia(id);
|
|
if (!row) return c.json({ ok: false, error: 'media not found' }, 404);
|
|
if (row.pinned) return c.json({ ok: false, error: 'Unpin this copy before evicting it.' }, 409);
|
|
if (!media?.evictNow) return c.json({ ok: false, error: 'media cache unavailable' }, 503);
|
|
await media.evictNow(id);
|
|
await audit(db, 'admin', 'cache.evict', id, { bytes: number(row.size) });
|
|
return c.json({ ok: true });
|
|
});
|
|
|
|
app.post('/api/admin/cache/:id/pin', adminAuth, async c => {
|
|
const id = c.req.param('id');
|
|
let body = {}; try { body = await c.req.json(); } catch {}
|
|
if (body.pinned !== true && body.pinned !== false) return c.json({ ok: false, error: 'pinned must be boolean' }, 400);
|
|
const target = await sql.execute({ sql: "SELECT size,pinned,status FROM media_cache WHERE video_id=?", args: [id] });
|
|
const row = target.rows[0];
|
|
if (!row) return c.json({ ok: false, error: 'media not found' }, 404);
|
|
if (body.pinned && row.status !== 'ready') return c.json({ ok: false, error: 'Only ready cache copies can be pinned.' }, 409);
|
|
const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3);
|
|
if (body.pinned && !row.pinned) {
|
|
const totals = await sql.execute('SELECT COALESCE(SUM(size),0) AS bytes FROM media_cache WHERE pinned=1');
|
|
if (number(totals.rows[0]?.bytes) + number(row.size) > pinnedBudget) return c.json({ ok: false, error: 'Pinned cache byte cap exceeded.' }, 409);
|
|
}
|
|
await sql.execute({ sql: 'UPDATE media_cache SET pinned=? WHERE video_id=?', args: [body.pinned ? 1 : 0, id] });
|
|
await audit(db, 'admin', body.pinned ? 'cache.pin' : 'cache.unpin', id, { bytes: number(row.size) });
|
|
return c.json({ ok: true, pinned: body.pinned });
|
|
});
|
|
|
|
app.get('/api/admin/index/health', adminAuth, async c => {
|
|
const [stats, sources, latest, backfill] = await Promise.all([
|
|
sql.execute(`SELECT COUNT(*) AS total,SUM(CASE WHEN url IS NOT NULL AND url<>'' THEN 1 ELSE 0 END) AS urls,
|
|
SUM(CASE WHEN EXISTS(SELECT 1 FROM video_details d WHERE d.video_id=video_meta.id) THEN 1 ELSE 0 END) AS details,
|
|
SUM(CASE WHEN EXISTS(SELECT 1 FROM video_thumbnails t WHERE t.video_id=video_meta.id AND t.data IS NOT NULL) THEN 1 ELSE 0 END) AS thumbnails FROM video_meta`),
|
|
sql.execute('SELECT source,COUNT(*) AS videos,SUM(discoveries) AS discoveries,MAX(last_seen) AS lastSeen FROM video_meta_sources GROUP BY source ORDER BY videos DESC'),
|
|
sql.execute('SELECT id,status,created_at,updated_at,state,error FROM deep_search_jobs ORDER BY created_at DESC LIMIT 1'),
|
|
sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source'),
|
|
]);
|
|
const row = stats.rows[0] || {}, total = number(row.total);
|
|
const last = latest.rows[0];
|
|
return c.json({ ok: true, total, urlCount: number(row.urls), detailCount: number(row.details), thumbnailCount: number(row.thumbnails),
|
|
urlCoverage: total ? number(row.urls) / total : 0, detailCoverage: total ? number(row.details) / total : 0, thumbnailCoverage: total ? number(row.thumbnails) / total : 0,
|
|
sources: sources.rows.map(x => ({ source: x.source, videos: number(x.videos), discoveries: number(x.discoveries), lastSeen: number(x.lastSeen) })),
|
|
lastRun: last ? { id: last.id, status: last.status, createdAt: number(last.created_at), updatedAt: number(last.updated_at), error: last.error || null, progress: parse(last.state) } : null,
|
|
backfill: backfill.rows.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) }, 200, { 'Cache-Control': 'no-store' });
|
|
});
|
|
|
|
app.post('/api/admin/index/backfill', adminAuth, async c => {
|
|
if (!backfillStep) return c.json({ ok: false, error: 'Catalog backfill is unavailable.' }, 503);
|
|
const processed = await backfillStep(100);
|
|
const state = (await sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source')).rows;
|
|
await audit(db, 'admin', 'index.backfill', '', { processed });
|
|
return c.json({ ok: true, processed, backfill: state.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) });
|
|
});
|
|
|
|
app.post('/api/admin/index/cleanup', adminAuth, async c => {
|
|
let body = {}; try { body = await c.req.json(); } catch {}
|
|
const days = Math.max(180, Math.min(3650, Number(body.days) || 365));
|
|
const cutoff = Date.now() - days * 86400_000;
|
|
const candidates = await sql.execute({ sql: `SELECT COUNT(*) AS n FROM video_meta m WHERE m.updated_at<?
|
|
AND NOT EXISTS(SELECT 1 FROM video_meta_sources s WHERE s.video_id=m.id)
|
|
AND NOT EXISTS(SELECT 1 FROM media_cache c WHERE c.video_id=m.id)
|
|
AND NOT EXISTS(SELECT 1 FROM listening_daily l WHERE l.video_id=m.id)`, args: [cutoff] });
|
|
const removed = number(candidates.rows[0]?.n);
|
|
await sql.execute({ sql: `DELETE FROM video_meta WHERE updated_at<?
|
|
AND NOT EXISTS(SELECT 1 FROM video_meta_sources s WHERE s.video_id=video_meta.id)
|
|
AND NOT EXISTS(SELECT 1 FROM media_cache c WHERE c.video_id=video_meta.id)
|
|
AND NOT EXISTS(SELECT 1 FROM listening_daily l WHERE l.video_id=video_meta.id)`, args: [cutoff] });
|
|
await sql.batch([
|
|
'DELETE FROM video_meta_sources WHERE video_id NOT IN (SELECT id FROM video_meta)',
|
|
'DELETE FROM video_thumbnails WHERE video_id NOT IN (SELECT id FROM video_meta)',
|
|
'DELETE FROM video_channels WHERE video_id NOT IN (SELECT id FROM video_meta)',
|
|
'DELETE FROM video_details WHERE video_id NOT IN (SELECT id FROM video_meta)',
|
|
], 'write');
|
|
await audit(db, 'admin', 'index.cleanup', '', { days, removed });
|
|
return c.json({ ok: true, removed, days });
|
|
});
|
|
|
|
}
|