import { audit } from './audit-log.js'; import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; const clientOf = db => typeof db?.execute === 'function' ? db : db?.db; const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } }; const number = value => Number(value) || 0; export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null, backfillStep = null }) { const sql = clientOf(db); const backupFile = date => join(backupDir || '.', `notes-${date}.json`); const backupDate = value => /^\d{4}-\d{2}-\d{2}$/.test(value || '') && !Number.isNaN(Date.parse(`${value}T00:00:00Z`)); async function readBackup(date) { if (!backupDate(date)) return null; try { const data = JSON.parse(await readFile(backupFile(date), 'utf8')); if (!Array.isArray(data.notes) || data.notes.some(note => !note?.videoId || !['lyrics', 'chapters'].includes(note.kind) || !note.data)) return null; return data; } catch { return null; } } async function createBackup({ safety = false } = {}) { if (!backupDir || typeof db.allNotes !== 'function') throw new Error('Backup storage unavailable'); await mkdir(safety ? join(backupDir, 'snapshots') : backupDir, { recursive: true }); const now = new Date(); const date = now.toISOString().slice(0, 10); const file = safety ? join(backupDir, 'snapshots', `notes-${date}-${now.toISOString().slice(11, 19).replaceAll(':', '-')}-safety.json`) : backupFile(date); const payload = { exportedAt: now.toISOString(), notes: await db.allNotes() }; await writeFile(file, JSON.stringify(payload)); return { file, date, notes: payload.notes.length }; } app.get('/api/admin/backups', adminAuth, async c => { if (!backupDir) return c.json({ ok: false, error: 'Backup storage unavailable' }, 503); let names = []; try { names = await readdir(backupDir); } catch {} const backups = []; for (const name of names.filter(value => /^notes-\d{4}-\d{2}-\d{2}\.json$/.test(value)).sort().reverse()) { const date = name.slice(6, 16); const data = await readBackup(date); const info = await stat(backupFile(date)).catch(() => null); backups.push({ date, bytes: info?.size || 0, exportedAt: data?.exportedAt || null, notes: data?.notes.length || 0, valid: !!data }); } return c.json({ ok: true, backups }, 200, { 'Cache-Control': 'no-store' }); }); app.get('/api/admin/backups/:date', adminAuth, async c => { const data = await readBackup(c.req.param('date')); if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404); const live = await db.allNotes(); const index = rows => new Map(rows.map(row => [`${row.videoId}:${row.kind}`, JSON.stringify(row.data)])); const old = index(data.notes), current = index(live); const added = [...old.keys()].filter(key => !current.has(key)); const removed = [...current.keys()].filter(key => !old.has(key)); const changed = [...old.keys()].filter(key => current.has(key) && old.get(key) !== current.get(key)); return c.json({ ok: true, date: c.req.param('date'), exportedAt: data.exportedAt, total: data.notes.length, diff: { added: added.length, removed: removed.length, changed: changed.length, preview: [...changed.slice(0, 30).map(key => ({ key, change: 'changed' })), ...added.slice(0, 15).map(key => ({ key, change: 'only in backup' })), ...removed.slice(0, 15).map(key => ({ key, change: 'only live' }))] } }); }); app.get('/api/admin/backups/:date/download', adminAuth, async c => { const data = await readBackup(c.req.param('date')); if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404); c.header('Content-Disposition', `attachment; filename="notes-${c.req.param('date')}.json"`); return c.json(data); }); app.post('/api/admin/backups', adminAuth, async c => { try { const result = await createBackup(); await audit(db, 'admin', 'backup.create', result.date, { notes: result.notes }); return c.json({ ok: true, ...result }); } catch (error) { return c.json({ ok: false, error: error.message }, 503); } }); app.post('/api/admin/backups/:date/restore', adminAuth, async c => { const date = c.req.param('date'); const data = await readBackup(date); if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404); let body = {}; try { body = await c.req.json(); } catch {} const selected = body.videoId ? data.notes.filter(note => note.videoId === body.videoId) : data.notes; if (body.videoId && !selected.length) return c.json({ ok: false, error: 'Song is not present in this backup' }, 404); const safety = await createBackup({ safety: true }); for (const note of selected) await db.saveNote({ videoId: note.videoId, kind: note.kind, data: note.data, source: 'restore', updatedBy: `admin (backup ${date})`, force: true }); if (!body.videoId) { const keep = new Set(data.notes.map(note => `${note.videoId}:${note.kind}`)); for (const note of await db.allNotes()) if (!keep.has(`${note.videoId}:${note.kind}`)) { await sql.execute({ sql: 'DELETE FROM video_notes WHERE video_id=? AND kind=?', args: [note.videoId, note.kind] }); } } await audit(db, 'admin', 'backup.restore', date, { videoId: body.videoId || null, restored: selected.length, safetySnapshot: safety.file }); return c.json({ ok: true, restored: selected.length, safetySnapshot: safety.file }); }); app.get('/api/admin/backups/export/sqlite-dump', adminAuth, async c => { const schema = await sql.execute("SELECT type,name,sql FROM sqlite_master WHERE type IN ('table','index','trigger') AND name NOT LIKE 'sqlite_%' AND sql IS NOT NULL ORDER BY CASE type WHEN 'table' THEN 0 ELSE 1 END,name"); const quote = value => { if (value == null) return 'NULL'; if (typeof value === 'number') return Number.isFinite(value) ? String(value) : 'NULL'; if (typeof value === 'bigint') return String(value); if (value instanceof Uint8Array) return `X'${Buffer.from(value).toString('hex')}'`; return `'${String(value).replaceAll("'", "''")}'`; }; const statements = ['PRAGMA foreign_keys=OFF;', 'BEGIN TRANSACTION;']; for (const table of schema.rows.filter(row => row.type === 'table')) { statements.push(`${table.sql};`); const rows = await sql.execute(`SELECT * FROM "${table.name.replaceAll('"', '""')}"`); for (const row of rows.rows) { const columns = Object.keys(row); if (!columns.length) continue; statements.push(`INSERT INTO "${table.name}" (${columns.map(name => `"${name.replaceAll('"', '""')}"`).join(',')}) VALUES (${columns.map(name => quote(row[name])).join(',')});`); } } for (const object of schema.rows.filter(row => row.type !== 'table')) statements.push(`${object.sql};`); statements.push('COMMIT;'); c.header('Content-Disposition', 'attachment; filename="ytplayer-notes-dump.sql"'); return c.text(statements.join('\n'), 200, { 'Content-Type': 'application/sql; charset=utf-8' }); }); app.get('/api/admin/lyrics/quality', adminAuth, async c => { const minLines = Math.max(1, Math.min(200, Number(c.req.query('minLines')) || 8)); const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50)); const category = ['all', 'missing', 'short', 'automatic', 'untimed', 'reports'].includes(c.req.query('category')) ? c.req.query('category') : 'all'; const conditions = { missing: 'n.video_id IS NULL', short: 'n.video_id IS NOT NULL AND COALESCE(json_array_length(json_extract(n.data,\'$.lines\')),0) < ?', automatic: "n.source='auto'", untimed: "EXISTS (SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL)", reports: "EXISTS (SELECT 1 FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open')", }; const parts = category === 'all' ? [conditions.missing, conditions.short, conditions.automatic, conditions.untimed, conditions.reports] : [conditions[category]]; const where = `WHERE (${parts.join(' OR ')})`; const args = category === 'short' ? [minLines] : category === 'all' ? [minLines] : []; const query = `SELECT m.id,m.card,m.updated_at,n.data AS lyrics,n.source AS lyricSource,n.updated_at AS lyricUpdated, COALESCE(json_array_length(json_extract(n.data,'$.lines')),0) AS lineCount, EXISTS(SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL) AS untimed, (SELECT COUNT(*) FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open') AS openReports FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics' ${where} ORDER BY openReports DESC,lyricUpdated ASC,m.updated_at DESC LIMIT ? OFFSET ?`; const rows = await sql.execute({ sql: query, args: [...args, limit, offset] }); const count = await sql.execute({ sql: `SELECT COUNT(*) AS total FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics' ${where}`, args }); const songs = rows.rows.map(row => { const card = parse(row.card); let untimed = !!row.untimed; let lines = number(row.lineCount); if (row.lyrics) { const data = parse(row.lyrics); lines ||= Array.isArray(data.lines) ? data.lines.length : 0; if (Array.isArray(data.lines)) untimed ||= data.lines.some(line => line.t == null); } const reasons = []; if (!row.lyrics) reasons.push('missing'); else if (lines < minLines) reasons.push('short'); if (row.lyricSource === 'auto') reasons.push('automatic'); if (untimed) reasons.push('untimed'); if (number(row.openReports)) reasons.push('reports'); return { id: row.id, title: card.title || row.id, channel: card.channel || '', lines, source: row.lyricSource || null, untimed, openReports: number(row.openReports), reasons, updatedAt: number(row.lyricUpdated) }; }); return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' }); }); app.get('/api/admin/lyrics/search', adminAuth, async c => { const query = String(c.req.query('q') || '').trim().slice(0, 120); const hasLyrics = ['yes', 'no', 'all'].includes(c.req.query('hasLyrics')) ? c.req.query('hasLyrics') : 'yes'; const hasChapters = ['yes', 'no', 'all'].includes(c.req.query('hasChapters')) ? c.req.query('hasChapters') : 'all'; const limit = Math.max(1, Math.min(50, Number(c.req.query('limit')) || 30)); const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); let hits = [], total = 0; if (query && hasLyrics !== 'no') { const chapterOptions = { hasChapters: hasChapters === 'yes', withoutChapters: hasChapters === 'no' }; hits = await db.searchNoteLyrics(query, { limit, offset, ...chapterOptions }); total = await db.countNoteLyrics(query, chapterOptions); } const terms = query.normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase().match(/[\p{L}\p{N}_-]+/gu) || []; if (!query || hasLyrics === 'no') { const parts = []; const args = []; if (hasLyrics === 'yes') parts.push("l.video_id IS NOT NULL AND l.data<>'{}'"); if (hasLyrics === 'no') parts.push('l.video_id IS NULL'); if (hasChapters === 'yes') parts.push("EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters')"); if (hasChapters === 'no') parts.push("NOT EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters')"); for (const term of terms) { parts.push('lower(m.hay) LIKE ?'); args.push(`%${term}%`); } const where = parts.length ? `WHERE ${parts.join(' AND ')}` : ''; const count = await sql.execute({ sql: `SELECT COUNT(*) AS n FROM video_meta m LEFT JOIN video_notes l ON l.video_id=m.id AND l.kind='lyrics' ${where}`, args }); const rows = await sql.execute({ sql: `SELECT m.id,m.card,l.data AS lyrics,l.rev,EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters') AS chapters FROM video_meta m LEFT JOIN video_notes l ON l.video_id=m.id AND l.kind='lyrics' ${where} ORDER BY m.updated_at DESC LIMIT ? OFFSET ?`, args: [...args, limit, offset] }); total = number(count.rows[0]?.n); hits = rows.rows.map(row => ({ videoId: row.id, snippet: parse(row.lyrics).lines?.slice(0, 3).map(line => line.text).join(' … ') || parse(row.card).title || row.id, rev: number(row.rev), hasChapters: !!row.chapters })); } if (hits.length && query && hasLyrics !== 'no') { const ids = hits.map(item => item.videoId); const rows = await sql.execute({ sql: `SELECT n.video_id,n.data,n.rev,m.card,EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=n.video_id AND ch.kind='chapters') AS chapters FROM video_notes n LEFT JOIN video_meta m ON m.id=n.video_id WHERE n.kind='lyrics' AND n.video_id IN (${ids.map(() => '?').join(',')})`, args: ids }); const byId = new Map(rows.rows.map(row => [row.video_id, row])); hits = hits.map(hit => { const row = byId.get(hit.videoId), card = parse(row?.card); return { ...hit, title: card.title || hit.videoId, channel: card.channel || '', rev: number(row?.rev), hasChapters: !!row?.chapters }; }); } return c.json({ ok: true, query, hasLyrics, hasChapters, total, limit, offset, results: hits }, 200, { 'Cache-Control': 'no-store' }); }); app.post('/api/admin/lyrics/search/rebuild', adminAuth, async c => { if (typeof db.rebuildLyricsSearch !== 'function') return c.json({ ok: false, error: 'Lyrics search rebuild is unavailable.' }, 503); const indexed = await db.rebuildLyricsSearch(); await audit(db, 'admin', 'lyrics.search.rebuild', '', { indexed }); return c.json({ ok: true, indexed }); }); app.get('/api/admin/cache/health', adminAuth, async c => { const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50)); const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); const rows = await sql.execute({ sql: "SELECT video_id,status,size,last_access,updated_at,gen,pinned FROM media_cache ORDER BY CASE WHEN status='ready' THEN 0 ELSE 1 END,last_access ASC LIMIT ? OFFSET ?", args: [limit, offset] }); const total = await sql.execute('SELECT COUNT(*) AS total FROM media_cache'); const grouped = await sql.execute('SELECT status,COUNT(*) AS count,COALESCE(SUM(size),0) AS bytes,COALESCE(SUM(CASE WHEN pinned=1 THEN size ELSE 0 END),0) AS pinnedBytes FROM media_cache GROUP BY status'); const bytes = grouped.rows.reduce((sum, row) => sum + number(row.bytes), 0); const pinnedBytes = grouped.rows.reduce((sum, row) => sum + number(row.pinnedBytes), 0); const budget = Math.max(0, Number(process.env.MEDIA_CACHE_MAX_BYTES) || 10 * 1024 ** 3); const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3); return c.json({ ok: true, budget, pinnedBudget, bytes, pinnedBytes, utilization: budget ? bytes / budget : 0, statuses: grouped.rows.map(row => ({ status: row.status, count: number(row.count), bytes: number(row.bytes) })), total: number(total.rows[0]?.total), limit, offset, media: rows.rows.map(row => ({ id: row.video_id, status: row.status, bytes: number(row.size), lastAccess: number(row.last_access), updatedAt: number(row.updated_at), gen: number(row.gen), pinned: !!row.pinned })) }, 200, { 'Cache-Control': 'no-store' }); }); app.post('/api/admin/cache/:id/evict', adminAuth, async c => { const id = c.req.param('id'); if (!/^[A-Za-z0-9_-]{6,64}$/.test(id)) return c.json({ ok: false, error: 'invalid media id' }, 400); const row = await db.getMedia(id); if (!row) return c.json({ ok: false, error: 'media not found' }, 404); if (row.pinned) return c.json({ ok: false, error: 'Unpin this copy before evicting it.' }, 409); if (!media?.evictNow) return c.json({ ok: false, error: 'media cache unavailable' }, 503); await media.evictNow(id); await audit(db, 'admin', 'cache.evict', id, { bytes: number(row.size) }); return c.json({ ok: true }); }); app.post('/api/admin/cache/:id/pin', adminAuth, async c => { const id = c.req.param('id'); let body = {}; try { body = await c.req.json(); } catch {} if (body.pinned !== true && body.pinned !== false) return c.json({ ok: false, error: 'pinned must be boolean' }, 400); const target = await sql.execute({ sql: "SELECT size,pinned,status FROM media_cache WHERE video_id=?", args: [id] }); const row = target.rows[0]; if (!row) return c.json({ ok: false, error: 'media not found' }, 404); if (body.pinned && row.status !== 'ready') return c.json({ ok: false, error: 'Only ready cache copies can be pinned.' }, 409); const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3); if (body.pinned && !row.pinned) { const totals = await sql.execute('SELECT COALESCE(SUM(size),0) AS bytes FROM media_cache WHERE pinned=1'); if (number(totals.rows[0]?.bytes) + number(row.size) > pinnedBudget) return c.json({ ok: false, error: 'Pinned cache byte cap exceeded.' }, 409); } await sql.execute({ sql: 'UPDATE media_cache SET pinned=? WHERE video_id=?', args: [body.pinned ? 1 : 0, id] }); await audit(db, 'admin', body.pinned ? 'cache.pin' : 'cache.unpin', id, { bytes: number(row.size) }); return c.json({ ok: true, pinned: body.pinned }); }); app.get('/api/admin/index/health', adminAuth, async c => { const [stats, sources, latest, backfill] = await Promise.all([ sql.execute(`SELECT COUNT(*) AS total,SUM(CASE WHEN url IS NOT NULL AND url<>'' THEN 1 ELSE 0 END) AS urls, SUM(CASE WHEN EXISTS(SELECT 1 FROM video_details d WHERE d.video_id=video_meta.id) THEN 1 ELSE 0 END) AS details, SUM(CASE WHEN EXISTS(SELECT 1 FROM video_thumbnails t WHERE t.video_id=video_meta.id AND t.data IS NOT NULL) THEN 1 ELSE 0 END) AS thumbnails FROM video_meta`), sql.execute('SELECT source,COUNT(*) AS videos,SUM(discoveries) AS discoveries,MAX(last_seen) AS lastSeen FROM video_meta_sources GROUP BY source ORDER BY videos DESC'), sql.execute('SELECT id,status,created_at,updated_at,state,error FROM deep_search_jobs ORDER BY created_at DESC LIMIT 1'), sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source'), ]); const row = stats.rows[0] || {}, total = number(row.total); const last = latest.rows[0]; return c.json({ ok: true, total, urlCount: number(row.urls), detailCount: number(row.details), thumbnailCount: number(row.thumbnails), urlCoverage: total ? number(row.urls) / total : 0, detailCoverage: total ? number(row.details) / total : 0, thumbnailCoverage: total ? number(row.thumbnails) / total : 0, sources: sources.rows.map(x => ({ source: x.source, videos: number(x.videos), discoveries: number(x.discoveries), lastSeen: number(x.lastSeen) })), lastRun: last ? { id: last.id, status: last.status, createdAt: number(last.created_at), updatedAt: number(last.updated_at), error: last.error || null, progress: parse(last.state) } : null, backfill: backfill.rows.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) }, 200, { 'Cache-Control': 'no-store' }); }); app.post('/api/admin/index/backfill', adminAuth, async c => { if (!backfillStep) return c.json({ ok: false, error: 'Catalog backfill is unavailable.' }, 503); const processed = await backfillStep(100); const state = (await sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source')).rows; await audit(db, 'admin', 'index.backfill', '', { processed }); return c.json({ ok: true, processed, backfill: state.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) }); }); app.post('/api/admin/index/cleanup', adminAuth, async c => { let body = {}; try { body = await c.req.json(); } catch {} const days = Math.max(180, Math.min(3650, Number(body.days) || 365)); const cutoff = Date.now() - days * 86400_000; const candidates = await sql.execute({ sql: `SELECT COUNT(*) AS n FROM video_meta m WHERE m.updated_at