From 180a28d3e429fe09d571327b5bf3a3d043c5bc42 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 01:56:53 +0800 Subject: [PATCH] Add a paged lyrics quality queue --- frontend/admin.html | 50 ++++++++++++++++++++++++++++- playwright.admin-audit.config.js | 2 +- server/admin-tools.js | 54 ++++++++++++++++++++++++++++++++ server/admin-tools.test.js | 37 ++++++++++++++++++++++ server/package.json | 2 +- server/server.js | 3 ++ server/transcriptions.js | 2 ++ tests/admin-quality.spec.js | 26 +++++++++++++++ 8 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 server/admin-tools.js create mode 100644 server/admin-tools.test.js create mode 100644 tests/admin-quality.spec.js diff --git a/frontend/admin.html b/frontend/admin.html index 1001d3b..256d2f9 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -542,6 +542,21 @@
+
+

Lyrics quality

+

Work the songs that need attention: missing, short, machine-drafted, untimed, or reported lyrics.

+
+
+ + + +
+

+
+
50 songs per page
+
+
+

Library

@@ -733,6 +748,7 @@ const ICONS = { editor: '', reports: '', + quality: '', library: '', peers: '', history: '', @@ -741,7 +757,7 @@ audit: '', access: '', }; - const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -756,6 +772,7 @@ if (push) history.replaceState(null, '', location.pathname + location.search + '#' + name); syncBars(); if (name === 'reports') loadReports(); + if (name === 'quality') loadLyricsQuality(); if (name === 'history') loadRevs(); if (name === 'peers') loadP2p(); if (name === 'library') loadUploads(); @@ -945,6 +962,37 @@ $('auditFilterForm').addEventListener('submit', event => { event.preventDefault(); auditOffset = 0; loadAudit(); }); $('auditPrev').addEventListener('click', () => { auditOffset = Math.max(0, auditOffset - 50); loadAudit(); }); $('auditNext').addEventListener('click', () => { auditOffset += 50; loadAudit(); }); + + /* ---------------- lyrics quality queue ---------------- */ + let qualityOffset = 0; + async function loadLyricsQuality() { + const table = $('qualityTable'); table.innerHTML = 'Loading quality queue…'; + const category = $('qualityCategory').value, minLines = Number($('qualityMinLines').value) || 8; + try { + const j = await api(`/api/admin/lyrics/quality?category=${encodeURIComponent(category)}&minLines=${minLines}&limit=50&offset=${qualityOffset}`); + if (!j.ok) throw Error(j.error || 'Could not load lyrics queue.'); + $('qualityCount').textContent = `${j.total.toLocaleString()} songs · ${j.songs.length ? qualityOffset + 1 : 0}–${qualityOffset + j.songs.length}`; + $('qualityPrev').disabled = qualityOffset === 0; $('qualityNext').disabled = qualityOffset + j.songs.length >= j.total; + table.innerHTML = j.songs.length ? `SongLyricsWhy hereActions${j.songs.map(song => `${esc(song.title)}
${esc(song.channel || song.id)}${song.lines} lines${song.source === 'auto' ? ' · auto draft' : ''}${song.untimed ? ' · untimed' : ''}${esc(song.reasons.join(' · '))}${song.openReports ? ` · ${song.openReports} reports` : ''}
`).join('')}` : 'No songs match this filter. Try All issues or reduce the minimum line count.'; + } catch (error) { table.innerHTML = `${esc(error.message)}`; } + } + $('qualityFilterForm').addEventListener('submit', event => { event.preventDefault(); qualityOffset = 0; loadLyricsQuality(); }); + $('qualityPrev').addEventListener('click', () => { qualityOffset = Math.max(0, qualityOffset - 50); loadLyricsQuality(); }); + $('qualityNext').addEventListener('click', () => { qualityOffset += 50; loadLyricsQuality(); }); + $('qualityTable').addEventListener('click', async event => { + const button = event.target.closest('button'); if (!button) return; + const id = button.dataset.qualityFix || button.dataset.qualityPull || button.dataset.qualityWhisper; + if (button.dataset.qualityFix) { leOpen(id); return; } + $('qualityStatus').className = 'msg'; $('qualityStatus').textContent = button.dataset.qualityPull ? 'Looking up LRCLIB…' : 'Adding to the Whisper queue…'; + try { + const result = button.dataset.qualityPull + ? await api(`/api/notes/${encodeURIComponent(id)}/lyrics/web`, { method: 'POST', body: {} }) + : await api(`/api/admin/transcriptions/${encodeURIComponent(id)}`, { method: 'POST', body: {} }); + if (!result.ok) throw Error(result.error || 'Action failed.'); + $('qualityStatus').className = 'msg ok'; $('qualityStatus').textContent = button.dataset.qualityPull ? `LRCLIB saved revision ${result.rev}.` : `Whisper ${result.job?.status || 'job queued'}.`; + loadLyricsQuality(); + } catch (error) { $('qualityStatus').className = 'msg err'; $('qualityStatus').textContent = error.message; } + }); $('tokenForm').addEventListener('submit', async (e) => { e.preventDefault(); const label = $('tokenLabel').value.trim(); diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js index 149dd36..9f1a999 100644 --- a/playwright.admin-audit.config.js +++ b/playwright.admin-audit.config.js @@ -1,3 +1,3 @@ const { defineConfig } = require('@playwright/test'); const classic = require('./playwright.classic.config'); -module.exports = defineConfig({ ...classic, testMatch: /admin-audit\.spec\.js/ }); +module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality)\.spec\.js/ }); diff --git a/server/admin-tools.js b/server/admin-tools.js new file mode 100644 index 0000000..72e2a5d --- /dev/null +++ b/server/admin-tools.js @@ -0,0 +1,54 @@ +import { audit } from './audit-log.js'; + +const clientOf = db => typeof db?.execute === 'function' ? db : db?.db; +const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } }; +const number = value => Number(value) || 0; + +export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null }) { + const sql = clientOf(db); + + app.get('/api/admin/lyrics/quality', adminAuth, async c => { + const minLines = Math.max(1, Math.min(200, Number(c.req.query('minLines')) || 8)); + const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); + const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50)); + const category = ['all', 'missing', 'short', 'automatic', 'untimed', 'reports'].includes(c.req.query('category')) ? c.req.query('category') : 'all'; + const conditions = { + missing: 'n.video_id IS NULL', + short: 'n.video_id IS NOT NULL AND COALESCE(json_array_length(json_extract(n.data,\'$.lines\')),0) < ?', + automatic: "n.source='auto'", + untimed: "EXISTS (SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL)", + reports: "EXISTS (SELECT 1 FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open')", + }; + const parts = category === 'all' + ? [conditions.missing, conditions.short, conditions.automatic, conditions.untimed, conditions.reports] + : [conditions[category]]; + const where = `WHERE (${parts.join(' OR ')})`; + const args = category === 'short' ? [minLines] : category === 'all' ? [minLines] : []; + const query = `SELECT m.id,m.card,m.updated_at,n.data AS lyrics,n.source AS lyricSource,n.updated_at AS lyricUpdated, + COALESCE(json_array_length(json_extract(n.data,'$.lines')),0) AS lineCount, + EXISTS(SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL) AS untimed, + (SELECT COUNT(*) FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open') AS openReports + FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics' + ${where} ORDER BY openReports DESC,lyricUpdated ASC,m.updated_at DESC LIMIT ? OFFSET ?`; + const rows = await sql.execute({ sql: query, args: [...args, limit, offset] }); + const count = await sql.execute({ sql: `SELECT COUNT(*) AS total FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics' ${where}`, args }); + const songs = rows.rows.map(row => { + const card = parse(row.card); + let untimed = !!row.untimed; + let lines = number(row.lineCount); + if (row.lyrics) { + const data = parse(row.lyrics); lines ||= Array.isArray(data.lines) ? data.lines.length : 0; + if (Array.isArray(data.lines)) untimed ||= data.lines.some(line => line.t == null); + } + const reasons = []; + if (!row.lyrics) reasons.push('missing'); + else if (lines < minLines) reasons.push('short'); + if (row.lyricSource === 'auto') reasons.push('automatic'); + if (untimed) reasons.push('untimed'); + if (number(row.openReports)) reasons.push('reports'); + return { id: row.id, title: card.title || row.id, channel: card.channel || '', lines, source: row.lyricSource || null, untimed, openReports: number(row.openReports), reasons, updatedAt: number(row.lyricUpdated) }; + }); + return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' }); + }); + +} diff --git a/server/admin-tools.test.js b/server/admin-tools.test.js new file mode 100644 index 0000000..3ae7189 --- /dev/null +++ b/server/admin-tools.test.js @@ -0,0 +1,37 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Hono } from 'hono'; + +const root = mkdtempSync(join(tmpdir(), 'ytp-admin-tools-')); +process.env.DB_PATH = join(root, 'admin.db'); +const db = await import('./db.js'); +const { registerAdminToolsRoutes } = await import('./admin-tools.js'); +const app = new Hono(); +const auth = async (c, next) => c.req.header('x-admin') === 'yes' ? next() : c.json({ ok: false }, 401); + +beforeAll(async () => { + await db.initDb(); + registerAdminToolsRoutes(app, { db, adminAuth: auth }); + await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] }); + await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] }); + await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] }); +}); +afterAll(() => rmSync(root, { recursive: true, force: true })); + +describe('lyrics quality queue', () => { + test('requires admin and returns missing, untimed and short-lyrics reasons in pages', async () => { + expect((await app.request('/api/admin/lyrics/quality')).status).toBe(401); + const headers = { 'x-admin': 'yes' }; + const missing = await (await app.request('/api/admin/lyrics/quality?category=missing', { headers })).json(); + expect(missing.total).toBe(1); + expect(missing.songs[0]).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] }); + const untimed = await (await app.request('/api/admin/lyrics/quality?category=untimed', { headers })).json(); + expect(untimed.songs[0]).toMatchObject({ id: 'Untimely001', lines: 2, untimed: true }); + const short = await (await app.request('/api/admin/lyrics/quality?category=short&minLines=3', { headers })).json(); + expect(short.songs[0].reasons).toContain('short'); + const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json(); + expect(page.songs).toHaveLength(1); + }); +}); diff --git a/server/package.json b/server/package.json index 248c5f4..0973e6b 100644 --- a/server/package.json +++ b/server/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "bun server.js", "dev": "bun --hot server.js", - "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./audit-log.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js" + "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./audit-log.test.js && bun test ./admin-tools.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js" }, "dependencies": { "@hono/node-server": "^1.14.0", diff --git a/server/server.js b/server/server.js index 376ec94..096869c 100644 --- a/server/server.js +++ b/server/server.js @@ -47,6 +47,7 @@ import { initDb, upsertUser, recordVideoAccess, getUserData, createProfile, getP import { createMediaCache, HIGH, LOW, validateMedia, MediaSkip } from './media-cache.js'; import * as notesDb from './db.js'; import { registerNoteRoutes, parseLrc, sanitizeLyrics } from './notes.js'; +import { registerAdminToolsRoutes } from './admin-tools.js'; import { createRemoteHub } from './remote.js'; import { createPartyHub } from './party.js'; import { registerMediaMetaRoutes } from './media-meta.js'; @@ -2317,6 +2318,8 @@ const notes = registerNoteRoutes(app, { const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, autoStart: false, searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads, ingest: collectVideoMetadata, putSearch: searchCacheDb.put }); +registerAdminToolsRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, media, + backupDir: process.env.UPLOAD_BACKUP_DIR || pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'backups') }); // ============================================================================ // Uploads — the server's own searchable media library (see uploads.js) diff --git a/server/transcriptions.js b/server/transcriptions.js index 8c2ab3d..4f24645 100644 --- a/server/transcriptions.js +++ b/server/transcriptions.js @@ -1,6 +1,7 @@ // Explicit Whisper requests produce reviewable drafts, never published notes. import { randomUUID } from 'node:crypto'; import { db as sql } from './db.js'; +import { audit } from './audit-log.js'; const ID = /^([\w-]{11}|upl_[a-f0-9]{12})$/; const LEASE_MS = 120000; function publicJob(row) { @@ -34,6 +35,7 @@ export function registerTranscriptionRoutes(app, { db, adminAuth, workerAuth, wo SELECT ?,?,?,?,? WHERE (SELECT COUNT(*) FROM lyric_transcriptions WHERE status IN ('queued','running')) < 25`, args: [randomUUID(), id, notes.lyrics?.rev || 0, time, time] }); const row = (await sql.execute({ sql: "SELECT * FROM lyric_transcriptions WHERE video_id=? AND status IN ('queued','running')", args: [id] })).rows[0]; + if (row) await audit(db, 'admin', 'lyrics.whisper.enqueue', id, { jobId: row.id }); return row ? c.json({ ok: true, job: publicJob(row) }, 202) : c.json({ ok: false, error: 'The transcription queue is full.' }, 429); }); app.get('/api/admin/transcriptions/:video', adminAuth, async c => { diff --git a/tests/admin-quality.spec.js b/tests/admin-quality.spec.js new file mode 100644 index 0000000..8ac1e7f --- /dev/null +++ b/tests/admin-quality.spec.js @@ -0,0 +1,26 @@ +const { test, expect } = require('@playwright/test'); + +test('lyrics queue exposes issue filters and reuses LRCLIB and Whisper routes', async ({ page }) => { + const calls = []; + await page.route('**/api/**', async route => { + const req = route.request(), url = new URL(req.url()); calls.push({ path: url.pathname, method: req.method() }); + let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true }; + if (url.pathname === '/api/admin/lyrics/quality') body = { ok: true, total: 1, offset: 0, songs: [{ id: 'QualMiss001', title: 'Missing lyric sample', channel: 'Choir', lines: 0, reasons: ['missing'], openReports: 0 }] }; + if (url.pathname === '/api/notes/QualMiss001/lyrics/web') body = { ok: true, rev: 2, lines: 12 }; + if (url.pathname === '/api/admin/transcriptions/QualMiss001') body = { ok: true, job: { status: 'queued' } }; + if (url.pathname === '/api/admin/media') body = { ok: true, media: [] }; + if (url.pathname === '/api/uploads') body = { ok: true, uploads: [] }; + if (url.pathname === '/api/admin/flags') body = { ok: true, open: 0, flags: [] }; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.goto('/admin.html#quality'); + await expect(page.locator('#p-quality')).toBeVisible(); + await expect(page.locator('#qualityTable')).toContainText('Missing lyric sample'); + await page.locator('#qualityCategory').selectOption('untimed'); await page.locator('#qualityFilterForm button').click(); + await expect(page.locator('#qualityTable')).toContainText('Pull LRCLIB'); + await page.locator('[data-quality-pull]').click(); await expect(page.locator('#qualityStatus')).toContainText('revision 2'); + await page.locator('[data-quality-whisper]').click(); await expect(page.locator('#qualityStatus')).toContainText('Whisper queued'); + expect(calls.some(x => x.path.endsWith('/lyrics/web') && x.method === 'POST')).toBe(true); + expect(calls.some(x => x.path.endsWith('/transcriptions/QualMiss001') && x.method === 'POST')).toBe(true); +});