Add a paged lyrics quality queue

server/admin-tools.js

@@ -0,0 +1,54 @@
+import { audit } from './audit-log.js';
+
+const clientOf = db => typeof db?.execute === 'function' ? db : db?.db;
+const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } };
+const number = value => Number(value) || 0;
+
+export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null }) {
+  const sql = clientOf(db);
+
+  app.get('/api/admin/lyrics/quality', adminAuth, async c => {
+    const minLines = Math.max(1, Math.min(200, Number(c.req.query('minLines')) || 8));
+    const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
+    const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50));
+    const category = ['all', 'missing', 'short', 'automatic', 'untimed', 'reports'].includes(c.req.query('category')) ? c.req.query('category') : 'all';
+    const conditions = {
+      missing: 'n.video_id IS NULL',
+      short: 'n.video_id IS NOT NULL AND COALESCE(json_array_length(json_extract(n.data,\'$.lines\')),0) < ?',
+      automatic: "n.source='auto'",
+      untimed: "EXISTS (SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL)",
+      reports: "EXISTS (SELECT 1 FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open')",
+    };
+    const parts = category === 'all'
+      ? [conditions.missing, conditions.short, conditions.automatic, conditions.untimed, conditions.reports]
+      : [conditions[category]];
+    const where = `WHERE (${parts.join(' OR ')})`;
+    const args = category === 'short' ? [minLines] : category === 'all' ? [minLines] : [];
+    const query = `SELECT m.id,m.card,m.updated_at,n.data AS lyrics,n.source AS lyricSource,n.updated_at AS lyricUpdated,
+      COALESCE(json_array_length(json_extract(n.data,'$.lines')),0) AS lineCount,
+      EXISTS(SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL) AS untimed,
+      (SELECT COUNT(*) FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open') AS openReports
+      FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics'
+      ${where} ORDER BY openReports DESC,lyricUpdated ASC,m.updated_at DESC LIMIT ? OFFSET ?`;
+    const rows = await sql.execute({ sql: query, args: [...args, limit, offset] });
+    const count = await sql.execute({ sql: `SELECT COUNT(*) AS total FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics' ${where}`, args });
+    const songs = rows.rows.map(row => {
+      const card = parse(row.card);
+      let untimed = !!row.untimed;
+      let lines = number(row.lineCount);
+      if (row.lyrics) {
+        const data = parse(row.lyrics); lines ||= Array.isArray(data.lines) ? data.lines.length : 0;
+        if (Array.isArray(data.lines)) untimed ||= data.lines.some(line => line.t == null);
+      }
+      const reasons = [];
+      if (!row.lyrics) reasons.push('missing');
+      else if (lines < minLines) reasons.push('short');
+      if (row.lyricSource === 'auto') reasons.push('automatic');
+      if (untimed) reasons.push('untimed');
+      if (number(row.openReports)) reasons.push('reports');
+      return { id: row.id, title: card.title || row.id, channel: card.channel || '', lines, source: row.lyricSource || null, untimed, openReports: number(row.openReports), reasons, updatedAt: number(row.lyricUpdated) };
+    });
+    return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' });
+  });
+
+}

server/admin-tools.test.js

@@ -0,0 +1,37 @@
+import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Hono } from 'hono';
+
+const root = mkdtempSync(join(tmpdir(), 'ytp-admin-tools-'));
+process.env.DB_PATH = join(root, 'admin.db');
+const db = await import('./db.js');
+const { registerAdminToolsRoutes } = await import('./admin-tools.js');
+const app = new Hono();
+const auth = async (c, next) => c.req.header('x-admin') === 'yes' ? next() : c.json({ ok: false }, 401);
+
+beforeAll(async () => {
+  await db.initDb();
+  registerAdminToolsRoutes(app, { db, adminAuth: auth });
+  await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] });
+  await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] });
+  await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] });
+});
+afterAll(() => rmSync(root, { recursive: true, force: true }));
+
+describe('lyrics quality queue', () => {
+  test('requires admin and returns missing, untimed and short-lyrics reasons in pages', async () => {
+    expect((await app.request('/api/admin/lyrics/quality')).status).toBe(401);
+    const headers = { 'x-admin': 'yes' };
+    const missing = await (await app.request('/api/admin/lyrics/quality?category=missing', { headers })).json();
+    expect(missing.total).toBe(1);
+    expect(missing.songs[0]).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] });
+    const untimed = await (await app.request('/api/admin/lyrics/quality?category=untimed', { headers })).json();
+    expect(untimed.songs[0]).toMatchObject({ id: 'Untimely001', lines: 2, untimed: true });
+    const short = await (await app.request('/api/admin/lyrics/quality?category=short&minLines=3', { headers })).json();
+    expect(short.songs[0].reasons).toContain('short');
+    const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json();
+    expect(page.songs).toHaveLength(1);
+  });
+});

server/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "start": "bun server.js",
     "dev": "bun --hot server.js",
-    "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./audit-log.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js"
+    "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./audit-log.test.js && bun test ./admin-tools.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js"
   },
   "dependencies": {
     "@hono/node-server": "^1.14.0",

server/server.js

@@ -47,6 +47,7 @@ import { initDb, upsertUser, recordVideoAccess, getUserData, createProfile, getP
 import { createMediaCache, HIGH, LOW, validateMedia, MediaSkip } from './media-cache.js';
 import * as notesDb from './db.js';
 import { registerNoteRoutes, parseLrc, sanitizeLyrics } from './notes.js';
+import { registerAdminToolsRoutes } from './admin-tools.js';
 import { createRemoteHub } from './remote.js';
 import { createPartyHub } from './party.js';
 import { registerMediaMetaRoutes } from './media-meta.js';
@@ -2317,6 +2318,8 @@ const notes = registerNoteRoutes(app, {
 const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, autoStart: false,
   searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads,
   ingest: collectVideoMetadata, putSearch: searchCacheDb.put });
+registerAdminToolsRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, media,
+  backupDir: process.env.UPLOAD_BACKUP_DIR || pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'backups') });
 
 // ============================================================================
 // Uploads — the server's own searchable media library (see uploads.js)

server/transcriptions.js

@@ -1,6 +1,7 @@
 // Explicit Whisper requests produce reviewable drafts, never published notes.
 import { randomUUID } from 'node:crypto';
 import { db as sql } from './db.js';
+import { audit } from './audit-log.js';
 const ID = /^([\w-]{11}|upl_[a-f0-9]{12})$/;
 const LEASE_MS = 120000;
 function publicJob(row) {
@@ -34,6 +35,7 @@ export function registerTranscriptionRoutes(app, { db, adminAuth, workerAuth, wo
       SELECT ?,?,?,?,? WHERE (SELECT COUNT(*) FROM lyric_transcriptions WHERE status IN ('queued','running')) < 25`,
       args: [randomUUID(), id, notes.lyrics?.rev || 0, time, time] });
     const row = (await sql.execute({ sql: "SELECT * FROM lyric_transcriptions WHERE video_id=? AND status IN ('queued','running')", args: [id] })).rows[0];
+    if (row) await audit(db, 'admin', 'lyrics.whisper.enqueue', id, { jobId: row.id });
     return row ? c.json({ ok: true, job: publicJob(row) }, 202) : c.json({ ok: false, error: 'The transcription queue is full.' }, 429);
   });
   app.get('/api/admin/transcriptions/:video', adminAuth, async c => {