469 lines
35 KiB
JavaScript
469 lines
35 KiB
JavaScript
import { audit } from './audit-log.js';
|
|
import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises';
|
|
import { basename, join } from 'node:path';
|
|
|
|
const clientOf = db => typeof db?.execute === 'function' ? db : db?.db;
|
|
const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } };
|
|
const number = value => Number(value) || 0;
|
|
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
const zipCrcTable = Uint32Array.from({ length: 256 }, (_, value) => {
|
|
let crc = value; for (let bit = 0; bit < 8; bit++) crc = crc & 1 ? 0xedb88320 ^ (crc >>> 1) : crc >>> 1;
|
|
return crc >>> 0;
|
|
});
|
|
function zipStore(files) {
|
|
const local = [], central = []; let offset = 0;
|
|
for (const file of files) {
|
|
const name = Buffer.from(file.name), data = Buffer.from(file.content);
|
|
let crc = 0xffffffff; for (const byte of data) crc = zipCrcTable[(crc ^ byte) & 0xff] ^ (crc >>> 8); crc = (crc ^ 0xffffffff) >>> 0;
|
|
const header = Buffer.alloc(30); header.writeUInt32LE(0x04034b50, 0); header.writeUInt16LE(20, 4); header.writeUInt16LE(0x0800, 6); header.writeUInt32LE(crc, 14); header.writeUInt32LE(data.length, 18); header.writeUInt32LE(data.length, 22); header.writeUInt16LE(name.length, 26);
|
|
local.push(header, name, data);
|
|
const directory = Buffer.alloc(46); directory.writeUInt32LE(0x02014b50, 0); directory.writeUInt16LE(20, 4); directory.writeUInt16LE(20, 6); directory.writeUInt16LE(0x0800, 8); directory.writeUInt32LE(crc, 16); directory.writeUInt32LE(data.length, 20); directory.writeUInt32LE(data.length, 24); directory.writeUInt16LE(name.length, 28); directory.writeUInt32LE(offset, 42);
|
|
central.push(directory, name); offset += header.length + name.length + data.length;
|
|
}
|
|
const centralBytes = Buffer.concat(central), localBytes = Buffer.concat(local), end = Buffer.alloc(22);
|
|
end.writeUInt32LE(0x06054b50, 0); end.writeUInt16LE(files.length, 8); end.writeUInt16LE(files.length, 10); end.writeUInt32LE(centralBytes.length, 12); end.writeUInt32LE(localBytes.length, 16);
|
|
return Buffer.concat([localBytes, centralBytes, end]);
|
|
}
|
|
|
|
export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null, backfillStep = null, pullLyrics = null, queueWhisper = null, autoStartJobs = true }) {
|
|
const sql = clientOf(db);
|
|
|
|
const backupFile = date => join(backupDir || '.', `notes-${date}.json`);
|
|
const backupDate = value => /^\d{4}-\d{2}-\d{2}$/.test(value || '') && !Number.isNaN(Date.parse(`${value}T00:00:00Z`));
|
|
async function readBackup(date) {
|
|
if (!backupDate(date)) return null;
|
|
try {
|
|
const data = JSON.parse(await readFile(backupFile(date), 'utf8'));
|
|
if (!Array.isArray(data.notes) || data.notes.some(note => !note?.videoId || !['lyrics', 'chapters'].includes(note.kind) || !note.data)) return null;
|
|
return data;
|
|
} catch { return null; }
|
|
}
|
|
async function createBackup({ safety = false } = {}) {
|
|
if (!backupDir || typeof db.allNotes !== 'function') throw new Error('Backup storage unavailable');
|
|
await mkdir(safety ? join(backupDir, 'snapshots') : backupDir, { recursive: true });
|
|
const now = new Date();
|
|
const date = now.toISOString().slice(0, 10);
|
|
const file = safety ? join(backupDir, 'snapshots', `notes-${date}-${now.toISOString().slice(11, 19).replaceAll(':', '-')}-safety.json`) : backupFile(date);
|
|
const payload = { exportedAt: now.toISOString(), notes: await db.allNotes() };
|
|
await writeFile(file, JSON.stringify(payload));
|
|
return { file, date, notes: payload.notes.length };
|
|
}
|
|
|
|
app.get('/api/admin/backups', adminAuth, async c => {
|
|
if (!backupDir) return c.json({ ok: false, error: 'Backup storage unavailable' }, 503);
|
|
let names = []; try { names = await readdir(backupDir); } catch {}
|
|
const backups = [];
|
|
for (const name of names.filter(value => /^notes-\d{4}-\d{2}-\d{2}\.json$/.test(value)).sort().reverse()) {
|
|
const date = name.slice(6, 16); const data = await readBackup(date); const info = await stat(backupFile(date)).catch(() => null);
|
|
backups.push({ date, bytes: info?.size || 0, exportedAt: data?.exportedAt || null, notes: data?.notes.length || 0, valid: !!data });
|
|
}
|
|
return c.json({ ok: true, backups }, 200, { 'Cache-Control': 'no-store' });
|
|
});
|
|
|
|
app.get('/api/admin/backups/:date', adminAuth, async c => {
|
|
const data = await readBackup(c.req.param('date'));
|
|
if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404);
|
|
const live = await db.allNotes();
|
|
const index = rows => new Map(rows.map(row => [`${row.videoId}:${row.kind}`, JSON.stringify(row.data)]));
|
|
const old = index(data.notes), current = index(live);
|
|
const added = [...old.keys()].filter(key => !current.has(key));
|
|
const removed = [...current.keys()].filter(key => !old.has(key));
|
|
const changed = [...old.keys()].filter(key => current.has(key) && old.get(key) !== current.get(key));
|
|
return c.json({ ok: true, date: c.req.param('date'), exportedAt: data.exportedAt, total: data.notes.length,
|
|
diff: { added: added.length, removed: removed.length, changed: changed.length, preview: [...changed.slice(0, 30).map(key => ({ key, change: 'changed' })), ...added.slice(0, 15).map(key => ({ key, change: 'only in backup' })), ...removed.slice(0, 15).map(key => ({ key, change: 'only live' }))] } });
|
|
});
|
|
|
|
app.get('/api/admin/backups/:date/download', adminAuth, async c => {
|
|
const data = await readBackup(c.req.param('date'));
|
|
if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404);
|
|
c.header('Content-Disposition', `attachment; filename="notes-${c.req.param('date')}.json"`);
|
|
return c.json(data);
|
|
});
|
|
|
|
app.post('/api/admin/backups', adminAuth, async c => {
|
|
try {
|
|
const result = await createBackup();
|
|
await audit(db, 'admin', 'backup.create', result.date, { notes: result.notes });
|
|
return c.json({ ok: true, ...result });
|
|
} catch (error) { return c.json({ ok: false, error: error.message }, 503); }
|
|
});
|
|
|
|
app.post('/api/admin/backups/:date/restore', adminAuth, async c => {
|
|
const date = c.req.param('date'); const data = await readBackup(date);
|
|
if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404);
|
|
let body = {}; try { body = await c.req.json(); } catch {}
|
|
const selected = body.videoId ? data.notes.filter(note => note.videoId === body.videoId) : data.notes;
|
|
if (body.videoId && !selected.length) return c.json({ ok: false, error: 'Song is not present in this backup' }, 404);
|
|
const safety = await createBackup({ safety: true });
|
|
for (const note of selected) await db.saveNote({ videoId: note.videoId, kind: note.kind, data: note.data, source: 'restore', updatedBy: `admin (backup ${date})`, force: true });
|
|
if (!body.videoId) {
|
|
const keep = new Set(data.notes.map(note => `${note.videoId}:${note.kind}`));
|
|
for (const note of await db.allNotes()) if (!keep.has(`${note.videoId}:${note.kind}`)) {
|
|
await sql.execute({ sql: 'DELETE FROM video_notes WHERE video_id=? AND kind=?', args: [note.videoId, note.kind] });
|
|
}
|
|
}
|
|
const safetySnapshot = basename(safety.file);
|
|
await audit(db, 'admin', 'backup.restore', date, { videoId: body.videoId || null, restored: selected.length, safetySnapshot });
|
|
return c.json({ ok: true, restored: selected.length, safetySnapshot });
|
|
});
|
|
|
|
app.get('/api/admin/backups/export/sqlite-dump', adminAuth, async c => {
|
|
const schema = await sql.execute("SELECT type,name,sql FROM sqlite_master WHERE type IN ('table','index','trigger') AND name NOT LIKE 'sqlite_%' AND sql IS NOT NULL ORDER BY CASE type WHEN 'table' THEN 0 ELSE 1 END,name");
|
|
const quote = value => {
|
|
if (value == null) return 'NULL';
|
|
if (typeof value === 'number') return Number.isFinite(value) ? String(value) : 'NULL';
|
|
if (typeof value === 'bigint') return String(value);
|
|
if (value instanceof Uint8Array) return `X'${Buffer.from(value).toString('hex')}'`;
|
|
return `'${String(value).replaceAll("'", "''")}'`;
|
|
};
|
|
const statements = ['PRAGMA foreign_keys=OFF;', 'BEGIN TRANSACTION;'];
|
|
for (const table of schema.rows.filter(row => row.type === 'table')) {
|
|
statements.push(`${table.sql};`);
|
|
const rows = await sql.execute(`SELECT * FROM "${table.name.replaceAll('"', '""')}"`);
|
|
for (const row of rows.rows) {
|
|
const columns = Object.keys(row);
|
|
if (!columns.length) continue;
|
|
statements.push(`INSERT INTO "${table.name}" (${columns.map(name => `"${name.replaceAll('"', '""')}"`).join(',')}) VALUES (${columns.map(name => quote(row[name])).join(',')});`);
|
|
}
|
|
}
|
|
for (const object of schema.rows.filter(row => row.type !== 'table')) statements.push(`${object.sql};`);
|
|
statements.push('COMMIT;');
|
|
c.header('Content-Disposition', 'attachment; filename="ytplayer-notes-dump.sql"');
|
|
return c.text(statements.join('\n'), 200, { 'Content-Type': 'application/sql; charset=utf-8' });
|
|
});
|
|
|
|
app.get('/api/admin/lyrics/quality', adminAuth, async c => {
|
|
const minLines = Math.max(1, Math.min(200, Number(c.req.query('minLines')) || 8));
|
|
const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
|
|
const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50));
|
|
const category = ['all', 'missing', 'short', 'automatic', 'untimed', 'reports'].includes(c.req.query('category')) ? c.req.query('category') : 'all';
|
|
const conditions = {
|
|
missing: 'n.video_id IS NULL',
|
|
short: 'n.video_id IS NOT NULL AND COALESCE(json_array_length(json_extract(n.data,\'$.lines\')),0) < ?',
|
|
automatic: "n.source='auto'",
|
|
untimed: "EXISTS (SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL)",
|
|
reports: "EXISTS (SELECT 1 FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open')",
|
|
};
|
|
const parts = category === 'all'
|
|
? [conditions.missing, conditions.short, conditions.automatic, conditions.untimed, conditions.reports]
|
|
: [conditions[category]];
|
|
const where = `WHERE (${parts.join(' OR ')})`;
|
|
const args = category === 'short' ? [minLines] : category === 'all' ? [minLines] : [];
|
|
const query = `SELECT m.id,m.card,m.updated_at,n.data AS lyrics,n.source AS lyricSource,n.updated_at AS lyricUpdated,
|
|
COALESCE(json_array_length(json_extract(n.data,'$.lines')),0) AS lineCount,
|
|
EXISTS(SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL) AS untimed,
|
|
(SELECT COUNT(*) FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open') AS openReports
|
|
FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics'
|
|
${where} ORDER BY openReports DESC,lyricUpdated ASC,m.updated_at DESC LIMIT ? OFFSET ?`;
|
|
const rows = await sql.execute({ sql: query, args: [...args, limit, offset] });
|
|
const count = await sql.execute({ sql: `SELECT COUNT(*) AS total FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics' ${where}`, args });
|
|
const songs = rows.rows.map(row => {
|
|
const card = parse(row.card);
|
|
let untimed = !!row.untimed;
|
|
let lines = number(row.lineCount);
|
|
if (row.lyrics) {
|
|
const data = parse(row.lyrics); lines ||= Array.isArray(data.lines) ? data.lines.length : 0;
|
|
if (Array.isArray(data.lines)) untimed ||= data.lines.some(line => line.t == null);
|
|
}
|
|
const reasons = [];
|
|
if (!row.lyrics) reasons.push('missing');
|
|
else if (lines < minLines) reasons.push('short');
|
|
if (row.lyricSource === 'auto') reasons.push('automatic');
|
|
if (untimed) reasons.push('untimed');
|
|
if (number(row.openReports)) reasons.push('reports');
|
|
return { id: row.id, title: card.title || row.id, channel: card.channel || '', lines, source: row.lyricSource || null, untimed, openReports: number(row.openReports), reasons, updatedAt: number(row.lyricUpdated) };
|
|
});
|
|
return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' });
|
|
});
|
|
|
|
app.get('/api/admin/lyrics/search', adminAuth, async c => {
|
|
const query = String(c.req.query('q') || '').trim().slice(0, 120);
|
|
const hasLyrics = ['yes', 'no', 'all'].includes(c.req.query('hasLyrics')) ? c.req.query('hasLyrics') : 'yes';
|
|
const hasChapters = ['yes', 'no', 'all'].includes(c.req.query('hasChapters')) ? c.req.query('hasChapters') : 'all';
|
|
const limit = Math.max(1, Math.min(50, Number(c.req.query('limit')) || 30));
|
|
const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
|
|
let hits = [], total = 0;
|
|
if (query && hasLyrics !== 'no') {
|
|
const chapterOptions = { hasChapters: hasChapters === 'yes', withoutChapters: hasChapters === 'no' };
|
|
hits = await db.searchNoteLyrics(query, { limit, offset, ...chapterOptions });
|
|
total = await db.countNoteLyrics(query, chapterOptions);
|
|
}
|
|
const terms = query.normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase().match(/[\p{L}\p{N}_-]+/gu) || [];
|
|
if (!query || hasLyrics === 'no') {
|
|
const parts = [];
|
|
const args = [];
|
|
if (hasLyrics === 'yes') parts.push("l.video_id IS NOT NULL AND l.data<>'{}'");
|
|
if (hasLyrics === 'no') parts.push('l.video_id IS NULL');
|
|
if (hasChapters === 'yes') parts.push("EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters')");
|
|
if (hasChapters === 'no') parts.push("NOT EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters')");
|
|
for (const term of terms) { parts.push('lower(m.hay) LIKE ?'); args.push(`%${term}%`); }
|
|
const where = parts.length ? `WHERE ${parts.join(' AND ')}` : '';
|
|
const count = await sql.execute({ sql: `SELECT COUNT(*) AS n FROM video_meta m LEFT JOIN video_notes l ON l.video_id=m.id AND l.kind='lyrics' ${where}`, args });
|
|
const rows = await sql.execute({ sql: `SELECT m.id,m.card,l.data AS lyrics,l.rev,EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters') AS chapters FROM video_meta m LEFT JOIN video_notes l ON l.video_id=m.id AND l.kind='lyrics' ${where} ORDER BY m.updated_at DESC LIMIT ? OFFSET ?`, args: [...args, limit, offset] });
|
|
total = number(count.rows[0]?.n);
|
|
hits = rows.rows.map(row => ({ videoId: row.id, snippet: parse(row.lyrics).lines?.slice(0, 3).map(line => line.text).join(' … ') || parse(row.card).title || row.id, rev: number(row.rev), hasChapters: !!row.chapters }));
|
|
}
|
|
if (hits.length && query && hasLyrics !== 'no') {
|
|
const ids = hits.map(item => item.videoId);
|
|
const rows = await sql.execute({ sql: `SELECT n.video_id,n.data,n.rev,m.card,EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=n.video_id AND ch.kind='chapters') AS chapters FROM video_notes n LEFT JOIN video_meta m ON m.id=n.video_id WHERE n.kind='lyrics' AND n.video_id IN (${ids.map(() => '?').join(',')})`, args: ids });
|
|
const byId = new Map(rows.rows.map(row => [row.video_id, row]));
|
|
hits = hits.map(hit => {
|
|
const row = byId.get(hit.videoId), card = parse(row?.card);
|
|
return { ...hit, title: card.title || hit.videoId, channel: card.channel || '', rev: number(row?.rev), hasChapters: !!row?.chapters };
|
|
});
|
|
}
|
|
return c.json({ ok: true, query, hasLyrics, hasChapters, total, limit, offset, results: hits }, 200, { 'Cache-Control': 'no-store' });
|
|
});
|
|
|
|
app.post('/api/admin/lyrics/search/rebuild', adminAuth, async c => {
|
|
if (typeof db.rebuildLyricsSearch !== 'function') return c.json({ ok: false, error: 'Lyrics search rebuild is unavailable.' }, 503);
|
|
const indexed = await db.rebuildLyricsSearch();
|
|
await audit(db, 'admin', 'lyrics.search.rebuild', '', { indexed });
|
|
return c.json({ ok: true, indexed });
|
|
});
|
|
|
|
const librarySelect = `SELECT id,title,channel,hay,lyrics,rev,cacheStatus FROM (
|
|
SELECT m.id,COALESCE(json_extract(m.card,'$.title'),m.id) AS title,COALESCE(json_extract(m.card,'$.channel'),'') AS channel,m.hay,
|
|
n.data AS lyrics,n.rev AS rev,c.status AS cacheStatus FROM video_meta m
|
|
LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics' LEFT JOIN media_cache c ON c.video_id=m.id
|
|
UNION ALL
|
|
SELECT u.id,u.title,COALESCE(u.artist,''),lower(u.title||' '||COALESCE(u.artist,'')),n.data,n.rev,NULL
|
|
FROM uploads u LEFT JOIN video_notes n ON n.video_id=u.id AND n.kind='lyrics'
|
|
)`;
|
|
function lyricsFilter(q) {
|
|
const words = String(q || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase().match(/[\p{L}\p{N}_-]+/gu) || [];
|
|
return { where: words.length ? `WHERE ${words.map(() => 'lower(hay) LIKE ?').join(' AND ')}` : '', args: words.map(word => `%${word}%`) };
|
|
}
|
|
app.get('/api/admin/lyrics/bulk/library', adminAuth, async c => {
|
|
const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50));
|
|
const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
|
|
const q = String(c.req.query('q') || '').slice(0, 120), filter = lyricsFilter(q);
|
|
const [count, rows] = await Promise.all([
|
|
sql.execute({ sql: `SELECT COUNT(*) AS total FROM (${librarySelect}) library ${filter.where}`, args: filter.args }),
|
|
sql.execute({ sql: `${librarySelect} ${filter.where} ORDER BY title COLLATE NOCASE,id LIMIT ? OFFSET ?`, args: [...filter.args, limit, offset] }),
|
|
]);
|
|
return c.json({ ok: true, q, total: number(count.rows[0]?.total), limit, offset, songs: rows.rows.map(row => {
|
|
const lyrics = parse(row.lyrics), lines = Array.isArray(lyrics.lines) ? lyrics.lines.length : 0;
|
|
return { id: row.id, title: row.title || row.id, channel: row.channel || '', lines, rev: number(row.rev), cacheStatus: row.cacheStatus || null };
|
|
}) }, 200, { 'Cache-Control': 'no-store' });
|
|
});
|
|
|
|
app.post('/api/admin/lyrics/jobs', adminAuth, async c => {
|
|
let body = {}; try { body = await c.req.json(); } catch {}
|
|
const action = ['lrclib', 'whisper', 'export', 'clear'].includes(body.action) ? body.action : '';
|
|
if (!action) return c.json({ ok: false, error: 'Choose LRCLIB, Whisper, export, or clear.' }, 400);
|
|
let ids = Array.isArray(body.ids) ? body.ids : [];
|
|
if (body.allFiltered === true) {
|
|
const filter = lyricsFilter(String(body.query || '').slice(0, 120));
|
|
const rows = await sql.execute({ sql: `${librarySelect} ${filter.where} ORDER BY title COLLATE NOCASE,id LIMIT 500`, args: filter.args });
|
|
ids = rows.rows.map(row => row.id);
|
|
}
|
|
ids = [...new Set(ids.map(String).filter(id => /^[\w-]{11}$/.test(id) || /^upl_[a-f0-9]{12}$/.test(id)))].slice(0, 500);
|
|
if (!ids.length) return c.json({ ok: false, error: 'Select at least one song.' }, 400);
|
|
if (action === 'clear' && body.confirmClear !== true) return c.json({ ok: false, error: 'Clearing lyrics requires explicit confirmation.' }, 400);
|
|
const id = crypto.randomUUID(), now = Math.floor(Date.now() / 1000);
|
|
const state = { done: 0, total: ids.length, results: [], logs: [], fileReady: false };
|
|
await sql.execute({ sql: 'INSERT INTO admin_lyrics_jobs(id,action,status,ids,state,created_at,updated_at) VALUES(?,?,\'queued\',?,?,?,?)', args: [id, action, JSON.stringify(ids), JSON.stringify(state), now, now] });
|
|
await audit(db, 'admin', 'lyrics.bulk.start', id, { action, count: ids.length });
|
|
return c.json({ ok: true, job: { id, action, status: 'queued', state } }, 202);
|
|
});
|
|
|
|
app.get('/api/admin/lyrics/jobs', adminAuth, async c => {
|
|
const rows = await sql.execute('SELECT * FROM admin_lyrics_jobs ORDER BY created_at DESC LIMIT 30');
|
|
return c.json({ ok: true, jobs: rows.rows.map(row => ({ id: row.id, action: row.action, status: row.status, state: parse(row.state), error: row.error, createdAt: number(row.created_at), updatedAt: number(row.updated_at) })) }, 200, { 'Cache-Control': 'no-store' });
|
|
});
|
|
app.get('/api/admin/lyrics/jobs/:id', adminAuth, async c => {
|
|
const row = (await sql.execute({ sql: 'SELECT * FROM admin_lyrics_jobs WHERE id=?', args: [c.req.param('id')] })).rows[0];
|
|
return row ? c.json({ ok: true, job: { id: row.id, action: row.action, status: row.status, state: parse(row.state), error: row.error, createdAt: number(row.created_at), updatedAt: number(row.updated_at) } }) : c.json({ ok: false, error: 'Job not found.' }, 404);
|
|
});
|
|
app.post('/api/admin/lyrics/jobs/:id/:action', adminAuth, async c => {
|
|
const id = c.req.param('id'), action = c.req.param('action');
|
|
if (!['pause', 'resume', 'cancel'].includes(action)) return c.json({ ok: false, error: 'Unknown job action.' }, 400);
|
|
const row = (await sql.execute({ sql: 'SELECT status FROM admin_lyrics_jobs WHERE id=?', args: [id] })).rows[0];
|
|
if (!row) return c.json({ ok: false, error: 'Job not found.' }, 404);
|
|
const next = action === 'pause' ? 'paused' : action === 'resume' ? 'queued' : 'cancelled';
|
|
if (action === 'pause' && !['queued', 'running'].includes(row.status) || action === 'resume' && row.status !== 'paused' || action === 'cancel' && !['queued', 'running', 'paused'].includes(row.status)) return c.json({ ok: false, error: `Cannot ${action} a ${row.status} job.` }, 409);
|
|
await sql.execute({ sql: 'UPDATE admin_lyrics_jobs SET status=?,updated_at=? WHERE id=?', args: [next, Math.floor(Date.now() / 1000), id] });
|
|
await audit(db, 'admin', `lyrics.bulk.${action}`, id, { from: row.status, to: next });
|
|
return c.json({ ok: true, status: next });
|
|
});
|
|
app.get('/api/admin/lyrics/jobs/:id/export', adminAuth, async c => {
|
|
const row = (await sql.execute({ sql: 'SELECT action,status,state FROM admin_lyrics_jobs WHERE id=?', args: [c.req.param('id')] })).rows[0];
|
|
const state = parse(row?.state);
|
|
if (!row || row.action !== 'export' || row.status !== 'complete' || !state.fileReady || !backupDir) return c.json({ ok: false, error: 'Export is not ready.' }, 404);
|
|
try { const data = await readFile(join(backupDir, 'bulk-lyrics', `${c.req.param('id')}.zip`)); return new Response(data, { headers: { 'Content-Type': 'application/zip', 'Content-Disposition': `attachment; filename="lyrics-${c.req.param('id')}.zip"` } }); }
|
|
catch { return c.json({ ok: false, error: 'Export file is unavailable.' }, 404); }
|
|
});
|
|
|
|
let workerBusy = false;
|
|
let activeJobId = null;
|
|
async function runBulkJob() {
|
|
if (workerBusy) return;
|
|
workerBusy = true;
|
|
try {
|
|
const row = (await sql.execute("SELECT * FROM admin_lyrics_jobs WHERE status IN ('queued','running') ORDER BY created_at,id LIMIT 1")).rows[0];
|
|
if (!row) return;
|
|
activeJobId = row.id;
|
|
await sql.execute({ sql: "UPDATE admin_lyrics_jobs SET status='running',updated_at=? WHERE id=? AND status='queued'", args: [Math.floor(Date.now() / 1000), row.id] });
|
|
const ids = parse(row.ids), state = parse(row.state); state.total = ids.length; state.results ||= []; state.logs ||= [];
|
|
for (let index = number(state.done); index < ids.length; index++) {
|
|
const current = (await sql.execute({ sql: 'SELECT status FROM admin_lyrics_jobs WHERE id=?', args: [row.id] })).rows[0]?.status;
|
|
if (current !== 'running') return;
|
|
const videoId = ids[index]; let result;
|
|
try {
|
|
if (row.action === 'lrclib') result = pullLyrics ? await pullLyrics(videoId) : { ok: false, error: 'LRCLIB worker is unavailable.' };
|
|
else if (row.action === 'whisper') {
|
|
const queued = queueWhisper ? await queueWhisper(videoId) : { status: 503, body: { ok: false, error: 'Whisper worker is unavailable.' } };
|
|
result = { ...(queued.body || {}), ok: queued.status < 400 && queued.body?.ok, httpStatus: queued.status };
|
|
} else if (row.action === 'clear') {
|
|
const saved = await db.saveNote({ videoId, kind: 'lyrics', data: { lines: [], tags: [], offset: 0 }, source: 'admin-bulk-clear', updatedBy: 'admin bulk clear', force: true });
|
|
const resolved = await db.autoResolveFlags(videoId, new Set());
|
|
await audit(db, 'admin', 'lyrics.clear', videoId, { rev: saved.rev, resolvedReports: resolved, undoViaRevision: saved.rev - 1 });
|
|
result = { ok: true, rev: saved.rev, resolvedReports: resolved };
|
|
} else {
|
|
const note = (await db.getNotes(videoId)).lyrics;
|
|
result = note ? { ok: true, lines: note.data.lines?.length || 0 } : { ok: false, error: 'No saved lyrics to export.' };
|
|
}
|
|
} catch (error) { result = { ok: false, error: String(error.message || error).slice(0, 300) }; }
|
|
const item = { id: videoId, ok: !!result?.ok, message: result?.ok ? (result.lines != null ? `${result.lines} lines` : 'Done') : result?.error || 'Action failed.' };
|
|
state.done = index + 1; state.results.push(item); state.logs.push({ at: Date.now(), ...item }); if (state.logs.length > 80) state.logs.splice(0, state.logs.length - 80);
|
|
await sql.execute({ sql: 'UPDATE admin_lyrics_jobs SET state=?,updated_at=? WHERE id=?', args: [JSON.stringify(state), Math.floor(Date.now() / 1000), row.id] });
|
|
if (row.action === 'lrclib') await sleep(1200);
|
|
}
|
|
if (row.action === 'export') {
|
|
const files = [];
|
|
for (const videoId of ids) {
|
|
const note = (await db.getNotes(videoId)).lyrics; if (!note?.data?.lines?.length) continue;
|
|
let title = videoId;
|
|
if (videoId.startsWith('upl_')) title = (await db.getUpload(videoId))?.title || videoId;
|
|
else {
|
|
const card = (await sql.execute({ sql: 'SELECT card FROM video_meta WHERE id=?', args: [videoId] })).rows[0]; title = parse(card?.card).title || videoId;
|
|
}
|
|
const safe = String(title).normalize('NFKC').replace(/[\\/:*?"<>|\u0000-\u001f]/g, '_').replace(/\s+/g, ' ').trim().slice(0, 100) || videoId;
|
|
const lrc = note.data.lines.map(line => line.t == null ? line.text : `[${String(Math.floor(line.t / 60)).padStart(2, '0')}:${(line.t % 60).toFixed(2).padStart(5, '0')}]${line.text}`).join('\n') + '\n';
|
|
files.push({ name: `${safe}__${videoId}.lrc`, content: lrc });
|
|
}
|
|
if (!backupDir) throw Error('Export storage is unavailable.');
|
|
await mkdir(join(backupDir, 'bulk-lyrics'), { recursive: true });
|
|
await writeFile(join(backupDir, 'bulk-lyrics', `${row.id}.zip`), zipStore(files));
|
|
state.fileReady = true; state.exportedFiles = files.length;
|
|
}
|
|
await sql.execute({ sql: "UPDATE admin_lyrics_jobs SET status='complete',state=?,updated_at=? WHERE id=? AND status='running'", args: [JSON.stringify(state), Math.floor(Date.now() / 1000), row.id] });
|
|
} catch (error) {
|
|
console.error('[admin-tools] lyrics job failed:', error.message);
|
|
if (activeJobId) {
|
|
try {
|
|
await sql.execute({ sql: "UPDATE admin_lyrics_jobs SET status='failed',error=?,updated_at=? WHERE id=? AND status='running'", args: [String(error.message || error).slice(0, 300), Math.floor(Date.now() / 1000), activeJobId] });
|
|
await audit(db, 'admin', 'lyrics.bulk.failed', activeJobId, { error: String(error.message || error).slice(0, 300) });
|
|
} catch (persistError) { console.warn('[admin-tools] could not persist lyrics job failure:', persistError.message); }
|
|
}
|
|
} finally { activeJobId = null; workerBusy = false; }
|
|
}
|
|
if (autoStartJobs) {
|
|
// The table is created by the schema setup, which may still be running at
|
|
// start-up: retry until it exists rather than skipping the resume.
|
|
const resume = (tries = 0) => sql.execute("UPDATE admin_lyrics_jobs SET status='queued',updated_at=unixepoch() WHERE status='running'").then(() => runBulkJob())
|
|
.catch(error => { if (/no such table/.test(error.message) && tries < 30) setTimeout(() => resume(tries + 1), 2000).unref?.(); else console.warn('[admin-tools] could not resume lyrics jobs:', error.message); });
|
|
resume();
|
|
const timer = setInterval(runBulkJob, 1000); timer.unref?.();
|
|
}
|
|
|
|
app.get('/api/admin/cache/health', adminAuth, async c => {
|
|
const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50));
|
|
const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
|
|
const rows = await sql.execute({ sql: "SELECT video_id,status,size,last_access,updated_at,gen,pinned FROM media_cache ORDER BY CASE WHEN status='ready' THEN 0 ELSE 1 END,last_access ASC LIMIT ? OFFSET ?", args: [limit, offset] });
|
|
const total = await sql.execute('SELECT COUNT(*) AS total FROM media_cache');
|
|
const grouped = await sql.execute('SELECT status,COUNT(*) AS count,COALESCE(SUM(size),0) AS bytes,COALESCE(SUM(CASE WHEN pinned=1 THEN size ELSE 0 END),0) AS pinnedBytes FROM media_cache GROUP BY status');
|
|
const bytes = grouped.rows.reduce((sum, row) => sum + number(row.bytes), 0);
|
|
const pinnedBytes = grouped.rows.reduce((sum, row) => sum + number(row.pinnedBytes), 0);
|
|
const budget = Math.max(0, Number(process.env.MEDIA_CACHE_MAX_BYTES) || 10 * 1024 ** 3);
|
|
const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3);
|
|
return c.json({ ok: true, budget, pinnedBudget, bytes, pinnedBytes, utilization: budget ? bytes / budget : 0,
|
|
statuses: grouped.rows.map(row => ({ status: row.status, count: number(row.count), bytes: number(row.bytes) })),
|
|
total: number(total.rows[0]?.total), limit, offset,
|
|
media: rows.rows.map(row => ({ id: row.video_id, status: row.status, bytes: number(row.size), lastAccess: number(row.last_access), updatedAt: number(row.updated_at), gen: number(row.gen), pinned: !!row.pinned })) }, 200, { 'Cache-Control': 'no-store' });
|
|
});
|
|
|
|
app.post('/api/admin/cache/:id/evict', adminAuth, async c => {
|
|
const id = c.req.param('id');
|
|
if (!/^[A-Za-z0-9_-]{6,64}$/.test(id)) return c.json({ ok: false, error: 'invalid media id' }, 400);
|
|
const row = await db.getMedia(id);
|
|
if (!row) return c.json({ ok: false, error: 'media not found' }, 404);
|
|
if (row.pinned) return c.json({ ok: false, error: 'Unpin this copy before evicting it.' }, 409);
|
|
if (!media?.evictNow) return c.json({ ok: false, error: 'media cache unavailable' }, 503);
|
|
await media.evictNow(id);
|
|
await audit(db, 'admin', 'cache.evict', id, { bytes: number(row.size) });
|
|
return c.json({ ok: true });
|
|
});
|
|
|
|
app.post('/api/admin/cache/:id/pin', adminAuth, async c => {
|
|
const id = c.req.param('id');
|
|
let body = {}; try { body = await c.req.json(); } catch {}
|
|
if (body.pinned !== true && body.pinned !== false) return c.json({ ok: false, error: 'pinned must be boolean' }, 400);
|
|
const target = await sql.execute({ sql: "SELECT size,pinned,status FROM media_cache WHERE video_id=?", args: [id] });
|
|
const row = target.rows[0];
|
|
if (!row) return c.json({ ok: false, error: 'media not found' }, 404);
|
|
if (body.pinned && row.status !== 'ready') return c.json({ ok: false, error: 'Only ready cache copies can be pinned.' }, 409);
|
|
const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3);
|
|
if (body.pinned && !row.pinned) {
|
|
const totals = await sql.execute('SELECT COALESCE(SUM(size),0) AS bytes FROM media_cache WHERE pinned=1');
|
|
if (number(totals.rows[0]?.bytes) + number(row.size) > pinnedBudget) return c.json({ ok: false, error: 'Pinned cache byte cap exceeded.' }, 409);
|
|
}
|
|
await sql.execute({ sql: 'UPDATE media_cache SET pinned=? WHERE video_id=?', args: [body.pinned ? 1 : 0, id] });
|
|
await audit(db, 'admin', body.pinned ? 'cache.pin' : 'cache.unpin', id, { bytes: number(row.size) });
|
|
return c.json({ ok: true, pinned: body.pinned });
|
|
});
|
|
|
|
app.get('/api/admin/index/health', adminAuth, async c => {
|
|
const [stats, sources, latest, backfill] = await Promise.all([
|
|
sql.execute(`SELECT COUNT(*) AS total,SUM(CASE WHEN url IS NOT NULL AND url<>'' THEN 1 ELSE 0 END) AS urls,
|
|
SUM(CASE WHEN EXISTS(SELECT 1 FROM video_details d WHERE d.video_id=video_meta.id) THEN 1 ELSE 0 END) AS details,
|
|
SUM(CASE WHEN EXISTS(SELECT 1 FROM video_thumbnails t WHERE t.video_id=video_meta.id AND t.data IS NOT NULL) THEN 1 ELSE 0 END) AS thumbnails FROM video_meta`),
|
|
sql.execute('SELECT source,COUNT(*) AS videos,SUM(discoveries) AS discoveries,MAX(last_seen) AS lastSeen FROM video_meta_sources GROUP BY source ORDER BY videos DESC'),
|
|
sql.execute('SELECT id,status,created_at,updated_at,state,error FROM deep_search_jobs ORDER BY created_at DESC LIMIT 1'),
|
|
sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source'),
|
|
]);
|
|
const row = stats.rows[0] || {}, total = number(row.total);
|
|
const last = latest.rows[0];
|
|
return c.json({ ok: true, total, urlCount: number(row.urls), detailCount: number(row.details), thumbnailCount: number(row.thumbnails),
|
|
urlCoverage: total ? number(row.urls) / total : 0, detailCoverage: total ? number(row.details) / total : 0, thumbnailCoverage: total ? number(row.thumbnails) / total : 0,
|
|
sources: sources.rows.map(x => ({ source: x.source, videos: number(x.videos), discoveries: number(x.discoveries), lastSeen: number(x.lastSeen) })),
|
|
lastRun: last ? { id: last.id, status: last.status, createdAt: number(last.created_at), updatedAt: number(last.updated_at), error: last.error || null, progress: parse(last.state) } : null,
|
|
backfill: backfill.rows.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) }, 200, { 'Cache-Control': 'no-store' });
|
|
});
|
|
|
|
app.post('/api/admin/index/backfill', adminAuth, async c => {
|
|
if (!backfillStep) return c.json({ ok: false, error: 'Catalog backfill is unavailable.' }, 503);
|
|
const processed = await backfillStep(100);
|
|
const state = (await sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source')).rows;
|
|
await audit(db, 'admin', 'index.backfill', '', { processed });
|
|
return c.json({ ok: true, processed, backfill: state.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) });
|
|
});
|
|
|
|
app.post('/api/admin/index/cleanup', adminAuth, async c => {
|
|
let body = {}; try { body = await c.req.json(); } catch {}
|
|
const days = Math.max(180, Math.min(3650, Number(body.days) || 365));
|
|
const cutoff = Date.now() - days * 86400_000;
|
|
const candidates = await sql.execute({ sql: `SELECT COUNT(*) AS n FROM video_meta m WHERE m.updated_at<?
|
|
AND NOT EXISTS(SELECT 1 FROM video_meta_sources s WHERE s.video_id=m.id)
|
|
AND NOT EXISTS(SELECT 1 FROM media_cache c WHERE c.video_id=m.id)
|
|
AND NOT EXISTS(SELECT 1 FROM listening_daily l WHERE l.video_id=m.id)`, args: [cutoff] });
|
|
const removed = number(candidates.rows[0]?.n);
|
|
await sql.execute({ sql: `DELETE FROM video_meta WHERE updated_at<?
|
|
AND NOT EXISTS(SELECT 1 FROM video_meta_sources s WHERE s.video_id=video_meta.id)
|
|
AND NOT EXISTS(SELECT 1 FROM media_cache c WHERE c.video_id=video_meta.id)
|
|
AND NOT EXISTS(SELECT 1 FROM listening_daily l WHERE l.video_id=video_meta.id)`, args: [cutoff] });
|
|
await sql.batch([
|
|
'DELETE FROM video_meta_sources WHERE video_id NOT IN (SELECT id FROM video_meta)',
|
|
'DELETE FROM video_thumbnails WHERE video_id NOT IN (SELECT id FROM video_meta)',
|
|
'DELETE FROM video_channels WHERE video_id NOT IN (SELECT id FROM video_meta)',
|
|
'DELETE FROM video_details WHERE video_id NOT IN (SELECT id FROM video_meta)',
|
|
], 'write');
|
|
await audit(db, 'admin', 'index.cleanup', '', { days, removed });
|
|
return c.json({ ok: true, removed, days });
|
|
});
|
|
|
|
return { runBulkJob };
|
|
|
|
}
|