Add a paged lyrics quality queue

2026-10-14 01:56:53 +08:00
parent 301fab335f
commit 180a28d3e4
8 changed files with 173 additions and 3 deletions
--- a/frontend/admin.html
+++ b/frontend/admin.html
@@ -542,6 +542,21 @@
      <div id="repList"></div>
    </section>

+    <section id="p-quality" class="panel">
+      <h1 class="page">Lyrics quality</h1>
+      <p class="lead">Work the songs that need attention: missing, short, machine-drafted, untimed, or reported lyrics.</p>
+      <div class="card">
+        <form id="qualityFilterForm" class="row">
+          <label>Issue<select class="field" id="qualityCategory"><option value="all">All issues</option><option value="missing">No lyrics</option><option value="short">Fewer than minimum lines</option><option value="automatic">Auto-transcribed, unreviewed</option><option value="untimed">Untimed lines</option><option value="reports">Open reports</option></select></label>
+          <label>Minimum lines<input class="field" id="qualityMinLines" type="number" min="1" max="200" value="8" /></label>
+          <button class="btn pri" type="submit">Update queue</button><span class="grow"></span><span id="qualityCount" class="muted" role="status" aria-live="polite"></span>
+        </form>
+        <p id="qualityStatus" class="msg" role="status" aria-live="polite"></p>
+        <div class="scroll"><table class="tbl" id="qualityTable"></table></div>
+        <div class="row"><button class="btn" id="qualityPrev" type="button">Previous</button><button class="btn" id="qualityNext" type="button">Next</button><span class="grow"></span><span class="muted">50 songs per page</span></div>
+      </div>
+    </section>
+
    <!-- ================= LIBRARY ================= -->
    <section id="p-library" class="panel">
      <h1 class="page">Library</h1>
@@ -733,6 +748,7 @@
  const ICONS = {
    editor: '<path d="M4 20h4L19 9a2.8 2.8 0 00-4-4L4 16v4z"/><path d="M13.5 6.5l4 4"/>',
    reports: '<path d="M5 21V4M5 4h12l-2.5 4L17 12H5"/>',
+    quality: '<path d="M4 5h16M4 12h16M4 19h10"/><path d="m17 18 2 2 4-5"/>',
    library: '<path d="M9 18V6l10-2v12"/><circle cx="6.5" cy="18" r="2.5"/><circle cx="16.5" cy="16" r="2.5"/>',
    peers: '<circle cx="6" cy="12" r="2.5"/><circle cx="18" cy="6" r="2.5"/><circle cx="18" cy="18" r="2.5"/><path d="M8.3 10.9l7.4-3.8M8.3 13.1l7.4 3.8"/>',
    history: '<path d="M3 12a9 9 0 109-9 9 9 0 00-6.4 2.6L3 8"/><path d="M3 3v5h5M12 7v5l3 2"/>',
@@ -741,7 +757,7 @@
    audit: '<path d="M5 4h14v16H5zM8 8h8M8 12h8M8 16h5"/>',
    access: '<circle cx="8" cy="15" r="4"/><path d="M11 12l9-9M16 7l3 3M14 9l2 2"/>',
  };
-  const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
+  const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
  let tab = 'editor';
  function buildTabs() {
    const html = TABS.map(([k, l]) => `<button class="tab" role="tab" data-tab="${k}" aria-selected="false"><svg viewBox="0 0 24 24">${ICONS[k]}</svg><span>${l}</span>${k === 'reports' ? '<span class="badge hidden" data-badge>0</span>' : ''}</button>`).join('');
@@ -756,6 +772,7 @@
    if (push) history.replaceState(null, '', location.pathname + location.search + '#' + name);
    syncBars();
    if (name === 'reports') loadReports();
+    if (name === 'quality') loadLyricsQuality();
    if (name === 'history') loadRevs();
    if (name === 'peers') loadP2p();
    if (name === 'library') loadUploads();
@@ -945,6 +962,37 @@
  $('auditFilterForm').addEventListener('submit', event => { event.preventDefault(); auditOffset = 0; loadAudit(); });
  $('auditPrev').addEventListener('click', () => { auditOffset = Math.max(0, auditOffset - 50); loadAudit(); });
  $('auditNext').addEventListener('click', () => { auditOffset += 50; loadAudit(); });
+
+  /* ---------------- lyrics quality queue ---------------- */
+  let qualityOffset = 0;
+  async function loadLyricsQuality() {
+    const table = $('qualityTable'); table.innerHTML = '<tr><td class="muted">Loading quality queue…</td></tr>';
+    const category = $('qualityCategory').value, minLines = Number($('qualityMinLines').value) || 8;
+    try {
+      const j = await api(`/api/admin/lyrics/quality?category=${encodeURIComponent(category)}&minLines=${minLines}&limit=50&offset=${qualityOffset}`);
+      if (!j.ok) throw Error(j.error || 'Could not load lyrics queue.');
+      $('qualityCount').textContent = `${j.total.toLocaleString()} songs · ${j.songs.length ? qualityOffset + 1 : 0}–${qualityOffset + j.songs.length}`;
+      $('qualityPrev').disabled = qualityOffset === 0; $('qualityNext').disabled = qualityOffset + j.songs.length >= j.total;
+      table.innerHTML = j.songs.length ? `<tr><th>Song</th><th>Lyrics</th><th>Why here</th><th>Actions</th></tr>${j.songs.map(song => `<tr><td><b>${esc(song.title)}</b><br><small>${esc(song.channel || song.id)}</small></td><td>${song.lines} lines${song.source === 'auto' ? ' · auto draft' : ''}${song.untimed ? ' · untimed' : ''}</td><td>${esc(song.reasons.join(' · '))}${song.openReports ? ` · ${song.openReports} reports` : ''}</td><td><div class="row"><button class="btn sm pri" data-quality-fix="${esc(song.id)}">Fix</button><button class="btn sm" data-quality-pull="${esc(song.id)}">Pull LRCLIB</button><button class="btn sm" data-quality-whisper="${esc(song.id)}">Queue Whisper</button></div></td></tr>`).join('')}` : '<tr><td class="muted">No songs match this filter. Try All issues or reduce the minimum line count.</td></tr>';
+    } catch (error) { table.innerHTML = `<tr><td class="err">${esc(error.message)}</td></tr>`; }
+  }
+  $('qualityFilterForm').addEventListener('submit', event => { event.preventDefault(); qualityOffset = 0; loadLyricsQuality(); });
+  $('qualityPrev').addEventListener('click', () => { qualityOffset = Math.max(0, qualityOffset - 50); loadLyricsQuality(); });
+  $('qualityNext').addEventListener('click', () => { qualityOffset += 50; loadLyricsQuality(); });
+  $('qualityTable').addEventListener('click', async event => {
+    const button = event.target.closest('button'); if (!button) return;
+    const id = button.dataset.qualityFix || button.dataset.qualityPull || button.dataset.qualityWhisper;
+    if (button.dataset.qualityFix) { leOpen(id); return; }
+    $('qualityStatus').className = 'msg'; $('qualityStatus').textContent = button.dataset.qualityPull ? 'Looking up LRCLIB…' : 'Adding to the Whisper queue…';
+    try {
+      const result = button.dataset.qualityPull
+        ? await api(`/api/notes/${encodeURIComponent(id)}/lyrics/web`, { method: 'POST', body: {} })
+        : await api(`/api/admin/transcriptions/${encodeURIComponent(id)}`, { method: 'POST', body: {} });
+      if (!result.ok) throw Error(result.error || 'Action failed.');
+      $('qualityStatus').className = 'msg ok'; $('qualityStatus').textContent = button.dataset.qualityPull ? `LRCLIB saved revision ${result.rev}.` : `Whisper ${result.job?.status || 'job queued'}.`;
+      loadLyricsQuality();
+    } catch (error) { $('qualityStatus').className = 'msg err'; $('qualityStatus').textContent = error.message; }
+  });
  $('tokenForm').addEventListener('submit', async (e) => {
    e.preventDefault();
    const label = $('tokenLabel').value.trim();
--- a/playwright.admin-audit.config.js
+++ b/playwright.admin-audit.config.js
@@ -1,3 +1,3 @@
 const { defineConfig } = require('@playwright/test');
 const classic = require('./playwright.classic.config');
-module.exports = defineConfig({ ...classic, testMatch: /admin-audit\.spec\.js/ });
+module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality)\.spec\.js/ });
--- a/server/admin-tools.js
+++ b/server/admin-tools.js
@@ -0,0 +1,54 @@
+import { audit } from './audit-log.js';
+
+const clientOf = db => typeof db?.execute === 'function' ? db : db?.db;
+const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } };
+const number = value => Number(value) || 0;
+
+export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null }) {
+  const sql = clientOf(db);
+
+  app.get('/api/admin/lyrics/quality', adminAuth, async c => {
+    const minLines = Math.max(1, Math.min(200, Number(c.req.query('minLines')) || 8));
+    const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
+    const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50));
+    const category = ['all', 'missing', 'short', 'automatic', 'untimed', 'reports'].includes(c.req.query('category')) ? c.req.query('category') : 'all';
+    const conditions = {
+      missing: 'n.video_id IS NULL',
+      short: 'n.video_id IS NOT NULL AND COALESCE(json_array_length(json_extract(n.data,\'$.lines\')),0) < ?',
+      automatic: "n.source='auto'",
+      untimed: "EXISTS (SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL)",
+      reports: "EXISTS (SELECT 1 FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open')",
+    };
+    const parts = category === 'all'
+      ? [conditions.missing, conditions.short, conditions.automatic, conditions.untimed, conditions.reports]
+      : [conditions[category]];
+    const where = `WHERE (${parts.join(' OR ')})`;
+    const args = category === 'short' ? [minLines] : category === 'all' ? [minLines] : [];
+    const query = `SELECT m.id,m.card,m.updated_at,n.data AS lyrics,n.source AS lyricSource,n.updated_at AS lyricUpdated,
+      COALESCE(json_array_length(json_extract(n.data,'$.lines')),0) AS lineCount,
+      EXISTS(SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL) AS untimed,
+      (SELECT COUNT(*) FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open') AS openReports
+      FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics'
+      ${where} ORDER BY openReports DESC,lyricUpdated ASC,m.updated_at DESC LIMIT ? OFFSET ?`;
+    const rows = await sql.execute({ sql: query, args: [...args, limit, offset] });
+    const count = await sql.execute({ sql: `SELECT COUNT(*) AS total FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics' ${where}`, args });
+    const songs = rows.rows.map(row => {
+      const card = parse(row.card);
+      let untimed = !!row.untimed;
+      let lines = number(row.lineCount);
+      if (row.lyrics) {
+        const data = parse(row.lyrics); lines ||= Array.isArray(data.lines) ? data.lines.length : 0;
+        if (Array.isArray(data.lines)) untimed ||= data.lines.some(line => line.t == null);
+      }
+      const reasons = [];
+      if (!row.lyrics) reasons.push('missing');
+      else if (lines < minLines) reasons.push('short');
+      if (row.lyricSource === 'auto') reasons.push('automatic');
+      if (untimed) reasons.push('untimed');
+      if (number(row.openReports)) reasons.push('reports');
+      return { id: row.id, title: card.title || row.id, channel: card.channel || '', lines, source: row.lyricSource || null, untimed, openReports: number(row.openReports), reasons, updatedAt: number(row.lyricUpdated) };
+    });
+    return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' });
+  });
+
+}
--- a/server/admin-tools.test.js
+++ b/server/admin-tools.test.js
@@ -0,0 +1,37 @@
+import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Hono } from 'hono';
+
+const root = mkdtempSync(join(tmpdir(), 'ytp-admin-tools-'));
+process.env.DB_PATH = join(root, 'admin.db');
+const db = await import('./db.js');
+const { registerAdminToolsRoutes } = await import('./admin-tools.js');
+const app = new Hono();
+const auth = async (c, next) => c.req.header('x-admin') === 'yes' ? next() : c.json({ ok: false }, 401);
+
+beforeAll(async () => {
+  await db.initDb();
+  registerAdminToolsRoutes(app, { db, adminAuth: auth });
+  await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] });
+  await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] });
+  await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] });
+});
+afterAll(() => rmSync(root, { recursive: true, force: true }));
+
+describe('lyrics quality queue', () => {
+  test('requires admin and returns missing, untimed and short-lyrics reasons in pages', async () => {
+    expect((await app.request('/api/admin/lyrics/quality')).status).toBe(401);
+    const headers = { 'x-admin': 'yes' };
+    const missing = await (await app.request('/api/admin/lyrics/quality?category=missing', { headers })).json();
+    expect(missing.total).toBe(1);
+    expect(missing.songs[0]).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] });
+    const untimed = await (await app.request('/api/admin/lyrics/quality?category=untimed', { headers })).json();
+    expect(untimed.songs[0]).toMatchObject({ id: 'Untimely001', lines: 2, untimed: true });
+    const short = await (await app.request('/api/admin/lyrics/quality?category=short&minLines=3', { headers })).json();
+    expect(short.songs[0].reasons).toContain('short');
+    const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json();
+    expect(page.songs).toHaveLength(1);
+  });
+});
--- a/server/package.json
+++ b/server/package.json
@@ -6,7 +6,7 @@
  "scripts": {
    "start": "bun server.js",
    "dev": "bun --hot server.js",
-    "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./audit-log.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js"
+    "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./audit-log.test.js && bun test ./admin-tools.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js"
  },
  "dependencies": {
    "@hono/node-server": "^1.14.0",
--- a/server/server.js
+++ b/server/server.js
@@ -47,6 +47,7 @@ import { initDb, upsertUser, recordVideoAccess, getUserData, createProfile, getP
 import { createMediaCache, HIGH, LOW, validateMedia, MediaSkip } from './media-cache.js';
 import * as notesDb from './db.js';
 import { registerNoteRoutes, parseLrc, sanitizeLyrics } from './notes.js';
+import { registerAdminToolsRoutes } from './admin-tools.js';
 import { createRemoteHub } from './remote.js';
 import { createPartyHub } from './party.js';
 import { registerMediaMetaRoutes } from './media-meta.js';
@@ -2317,6 +2318,8 @@ const notes = registerNoteRoutes(app, {
 const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, autoStart: false,
  searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads,
  ingest: collectVideoMetadata, putSearch: searchCacheDb.put });
+registerAdminToolsRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, media,
+  backupDir: process.env.UPLOAD_BACKUP_DIR || pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'backups') });

 // ============================================================================
 // Uploads — the server's own searchable media library (see uploads.js)
--- a/server/transcriptions.js
+++ b/server/transcriptions.js
@@ -1,6 +1,7 @@
 // Explicit Whisper requests produce reviewable drafts, never published notes.
 import { randomUUID } from 'node:crypto';
 import { db as sql } from './db.js';
+import { audit } from './audit-log.js';
 const ID = /^([\w-]{11}|upl_[a-f0-9]{12})$/;
 const LEASE_MS = 120000;
 function publicJob(row) {
@@ -34,6 +35,7 @@ export function registerTranscriptionRoutes(app, { db, adminAuth, workerAuth, wo
      SELECT ?,?,?,?,? WHERE (SELECT COUNT(*) FROM lyric_transcriptions WHERE status IN ('queued','running')) < 25`,
      args: [randomUUID(), id, notes.lyrics?.rev || 0, time, time] });
    const row = (await sql.execute({ sql: "SELECT * FROM lyric_transcriptions WHERE video_id=? AND status IN ('queued','running')", args: [id] })).rows[0];
+    if (row) await audit(db, 'admin', 'lyrics.whisper.enqueue', id, { jobId: row.id });
    return row ? c.json({ ok: true, job: publicJob(row) }, 202) : c.json({ ok: false, error: 'The transcription queue is full.' }, 429);
  });
  app.get('/api/admin/transcriptions/:video', adminAuth, async c => {
--- a/tests/admin-quality.spec.js
+++ b/tests/admin-quality.spec.js
@@ -0,0 +1,26 @@
+const { test, expect } = require('@playwright/test');
+
+test('lyrics queue exposes issue filters and reuses LRCLIB and Whisper routes', async ({ page }) => {
+  const calls = [];
+  await page.route('**/api/**', async route => {
+    const req = route.request(), url = new URL(req.url()); calls.push({ path: url.pathname, method: req.method() });
+    let body = { ok: true };
+    if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true };
+    if (url.pathname === '/api/admin/lyrics/quality') body = { ok: true, total: 1, offset: 0, songs: [{ id: 'QualMiss001', title: 'Missing lyric sample', channel: 'Choir', lines: 0, reasons: ['missing'], openReports: 0 }] };
+    if (url.pathname === '/api/notes/QualMiss001/lyrics/web') body = { ok: true, rev: 2, lines: 12 };
+    if (url.pathname === '/api/admin/transcriptions/QualMiss001') body = { ok: true, job: { status: 'queued' } };
+    if (url.pathname === '/api/admin/media') body = { ok: true, media: [] };
+    if (url.pathname === '/api/uploads') body = { ok: true, uploads: [] };
+    if (url.pathname === '/api/admin/flags') body = { ok: true, open: 0, flags: [] };
+    await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) });
+  });
+  await page.goto('/admin.html#quality');
+  await expect(page.locator('#p-quality')).toBeVisible();
+  await expect(page.locator('#qualityTable')).toContainText('Missing lyric sample');
+  await page.locator('#qualityCategory').selectOption('untimed'); await page.locator('#qualityFilterForm button').click();
+  await expect(page.locator('#qualityTable')).toContainText('Pull LRCLIB');
+  await page.locator('[data-quality-pull]').click(); await expect(page.locator('#qualityStatus')).toContainText('revision 2');
+  await page.locator('[data-quality-whisper]').click(); await expect(page.locator('#qualityStatus')).toContainText('Whisper queued');
+  expect(calls.some(x => x.path.endsWith('/lyrics/web') && x.method === 'POST')).toBe(true);
+  expect(calls.some(x => x.path.endsWith('/transcriptions/QualMiss001') && x.method === 'POST')).toBe(true);
+});