From 301fab335f65f05ea671a3fd846232601984ffd7 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 01:54:08 +0800 Subject: [PATCH 1/9] Add searchable admin audit history --- frontend/admin.html | 58 +++++++++++++++++++++++++++++++- playwright.admin-audit.config.js | 3 ++ server/audit-log.js | 53 +++++++++++++++++++++++++++++ server/audit-log.test.js | 35 +++++++++++++++++++ server/db.js | 11 ++++++ server/deep-search.js | 3 ++ server/notes.js | 20 +++++++++-- server/package.json | 2 +- tests/admin-audit.spec.js | 27 +++++++++++++++ 9 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 playwright.admin-audit.config.js create mode 100644 server/audit-log.js create mode 100644 server/audit-log.test.js create mode 100644 tests/admin-audit.spec.js diff --git a/frontend/admin.html b/frontend/admin.html index 5743cfa..1001d3b 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -115,6 +115,11 @@ pre { margin: 0; padding: 14px; border-radius: 14px; background: rgba(0,0,0,.35); border: 1px solid var(--line); overflow: auto; font: 12px/1.55 var(--mono); white-space: pre-wrap; word-break: break-word; max-height: 60vh; } .newtoken { margin-top: 12px; padding: 12px; border: 1px solid var(--ok); border-radius: 12px; background: rgba(74,222,128,.07); word-break: break-all; } .src { font: 11px var(--mono); color: var(--text-dim); } .src.api, .src.auto { color: var(--warn); } .src.restore { color: var(--ok); } + .audit-filters { display:grid; grid-template-columns:repeat(5,minmax(0,1fr)); gap:10px; align-items:end; } + .audit-filters label { display:grid; gap:5px; color:var(--text-2); font-size:12px; } + .audit-tools { display:flex; gap:8px; align-items:center; flex-wrap:wrap; } + .audit-detail { color:var(--text-2); font:12px/1.4 var(--mono); max-width:36ch; overflow-wrap:anywhere; } + @media(max-width:720px) { .audit-filters { grid-template-columns:repeat(2,minmax(0,1fr)); } } /* ---------- upload card ---------- */ .sr-file { position: absolute; width: 1px; height: 1px; opacity: 0; overflow: hidden; clip: rect(0 0 0 0); } @@ -659,6 +664,24 @@

Runs

+
+

Audit log

+

A searchable record of admin changes, with the actor and target kept alongside each action.

+
+
+ + + + + +
Export JSONExport CSV
+ +
+
+
50 events per page
+
+
+

API access

Tokens let scripts write shared lyrics and chapters. A token is shown once — store it in your secrets vault.

@@ -715,9 +738,10 @@ history: '', analytics: '', 'deep-search': '', + audit: '', access: '', }; - const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -738,6 +762,7 @@ if (name === 'access') loadTokens(); if (name === 'analytics') loadAnalytics(); if (name === 'deep-search') loadDeepSearch(); + if (name === 'audit') loadAudit(); window.scrollTo({ top: 0 }); } function setBadge(n) { @@ -889,6 +914,37 @@ const rows = (j.tokens || []).map((t) => `${esc(t.label)}${when(t.createdAt)}${t.lastUsedAt ? when(t.lastUsedAt) : 'never'}`).join(''); $('tokenTable').innerHTML = rows ? `LabelCreatedLast used${rows}` : 'No tokens yet.'; } + + /* ---------------- audit history ---------------- */ + let auditOffset = 0; + function auditQuery(format = '') { + const p = new URLSearchParams(format ? {} : { limit: '50', offset: String(auditOffset) }); + const search = $('auditSearch').value.trim(); if (search) p.set('q', search); + for (const [key, id] of [['actor', 'auditActor'], ['action', 'auditAction']]) { + const value = $(id).value.trim(); if (value) p.set(key, value); + } + for (const [key, id, end] of [['from', 'auditFrom', false], ['to', 'auditTo', true]]) { + const value = $(id).value; + if (value) p.set(key, String(Math.floor(new Date(`${value}T${end ? '23:59:59' : '00:00:00'}Z`).getTime() / 1000))); + } + if (format) p.set('format', format); + return p.toString(); + } + async function loadAudit() { + const table = $('auditTable'); table.innerHTML = 'Loading audit events…'; + try { + const j = await api('/api/admin/audit?' + auditQuery()); + if (!j.ok) throw Error(j.error || 'Audit log unavailable.'); + const rows = j.events.map(event => `${when(event.ts)}${esc(event.actor)}${esc(event.action)}${esc(event.target || '—')}${esc(JSON.stringify(event.detail || {}))}`).join(''); + table.innerHTML = rows ? `WhenActorActionTargetDetails${rows}` : 'No matching events.'; + $('auditCount').textContent = `${j.total.toLocaleString()} events · ${j.events.length ? auditOffset + 1 : 0}–${auditOffset + j.events.length}`; + $('auditPrev').disabled = auditOffset === 0; $('auditNext').disabled = auditOffset + j.events.length >= j.total; + $('auditJson').href = '/api/admin/audit?' + auditQuery('json'); $('auditCsv').href = '/api/admin/audit?' + auditQuery('csv'); + } catch (error) { table.innerHTML = `${esc(error.message)}`; } + } + $('auditFilterForm').addEventListener('submit', event => { event.preventDefault(); auditOffset = 0; loadAudit(); }); + $('auditPrev').addEventListener('click', () => { auditOffset = Math.max(0, auditOffset - 50); loadAudit(); }); + $('auditNext').addEventListener('click', () => { auditOffset += 50; loadAudit(); }); $('tokenForm').addEventListener('submit', async (e) => { e.preventDefault(); const label = $('tokenLabel').value.trim(); diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js new file mode 100644 index 0000000..149dd36 --- /dev/null +++ b/playwright.admin-audit.config.js @@ -0,0 +1,3 @@ +const { defineConfig } = require('@playwright/test'); +const classic = require('./playwright.classic.config'); +module.exports = defineConfig({ ...classic, testMatch: /admin-audit\.spec\.js/ }); diff --git a/server/audit-log.js b/server/audit-log.js new file mode 100644 index 0000000..4ed78c0 --- /dev/null +++ b/server/audit-log.js @@ -0,0 +1,53 @@ +const asJson = value => { + try { return JSON.stringify(value ?? {}); } catch { return '{}'; } +} + +export async function audit(db, actor, action, target = '', detail = {}) { + const client = typeof db?.execute === 'function' ? db : db?.db; + if (!client?.execute) throw new TypeError('audit database client is unavailable'); + await client.execute({ + sql: 'INSERT INTO audit_log (ts,actor,action,target,detail) VALUES (unixepoch(),?,?,?,?)', + args: [String(actor || 'admin').slice(0, 120), String(action).slice(0, 120), String(target || '').slice(0, 300), asJson(detail).slice(0, 8000)], + }); +} + +export function registerAuditRoutes(app, { db, adminAuth }) { + const client = typeof db?.execute === 'function' ? db : db?.db; + app.get('/api/admin/audit', adminAuth, async c => { + const format = c.req.query('format'); + const limit = Math.max(1, Math.min(format ? 10000 : 200, Number(c.req.query('limit')) || (format ? 10000 : 50))); + const offset = format ? 0 : Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); + const filters = []; + const args = []; + for (const key of ['actor', 'action']) { + const value = String(c.req.query(key) || '').trim().slice(0, 120); + if (value) { filters.push(`${key} LIKE ?`); args.push(`%${value.replace(/[\\%_]/g, '\\$&')}%`); } + } + const search = String(c.req.query('q') || '').trim().slice(0, 160); + if (search) { filters.push('(actor LIKE ? OR action LIKE ? OR target LIKE ? OR detail LIKE ?)'); args.push(...Array(4).fill(`%${search.replace(/[\\%_]/g, '\\$&')}%`)); } + const from = Number(c.req.query('from')); + const to = Number(c.req.query('to')); + if (Number.isFinite(from) && from > 0) { filters.push('ts >= ?'); args.push(Math.floor(from)); } + if (Number.isFinite(to) && to > 0) { filters.push('ts <= ?'); args.push(Math.floor(to)); } + const where = filters.length ? `WHERE ${filters.join(' AND ')}` : ''; + const [rows, count] = await Promise.all([ + client.execute({ sql: `SELECT id,ts,actor,action,target,detail FROM audit_log ${where} ORDER BY ts DESC,id DESC LIMIT ? OFFSET ?`, args: [...args, limit, offset] }), + client.execute({ sql: `SELECT COUNT(*) AS total FROM audit_log ${where}`, args }), + ]); + const events = rows.rows.map(row => ({ + id: Number(row.id), ts: Number(row.ts), actor: row.actor, action: row.action, target: row.target, + detail: (() => { try { return JSON.parse(row.detail || '{}'); } catch { return {}; } })(), + })); + if (format === 'json') { + c.header('Content-Disposition', 'attachment; filename="ytplayer-audit.json"'); + return c.json({ events, total: Number(count.rows[0]?.total) || 0 }); + } + if (format === 'csv') { + const quote = value => `"${String(value ?? '').replaceAll('"', '""')}"`; + const body = ['id,ts,actor,action,target,detail', ...events.map(row => [row.id, new Date(row.ts * 1000).toISOString(), row.actor, row.action, row.target, JSON.stringify(row.detail)].map(quote).join(','))].join('\n'); + c.header('Content-Disposition', 'attachment; filename="ytplayer-audit.csv"'); + return c.text(body, 200, { 'Content-Type': 'text/csv; charset=utf-8' }); + } + return c.json({ ok: true, events, total: Number(count.rows[0]?.total) || 0, limit, offset }, 200, { 'Cache-Control': 'no-store' }); + }); +} diff --git a/server/audit-log.test.js b/server/audit-log.test.js new file mode 100644 index 0000000..7a626fc --- /dev/null +++ b/server/audit-log.test.js @@ -0,0 +1,35 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Hono } from 'hono'; + +const root = mkdtempSync(join(tmpdir(), 'ytp-audit-test-')); +process.env.DB_PATH = join(root, 'audit.db'); +const db = await import('./db.js'); +const { audit, registerAuditRoutes } = await import('./audit-log.js'); +const app = new Hono(); +const auth = async (c, next) => c.req.header('x-admin') === 'yes' ? next() : c.json({ ok: false }, 401); + +beforeAll(async () => { + await db.initDb(); + registerAuditRoutes(app, { db, adminAuth: auth }); + await audit(db, 'admin', 'notes.save.lyrics', 'video_1', { rev: 2 }); + await audit(db, 'editor@example.test', 'report.resolve', '42', { batch: true }); +}); +afterAll(() => rmSync(root, { recursive: true, force: true })); + +describe('admin audit log', () => { + test('is authenticated, filtered, paged and exports CSV/JSON', async () => { + expect((await app.request('/api/admin/audit')).status).toBe(401); + const headers = { 'x-admin': 'yes' }; + const filtered = await (await app.request('/api/admin/audit?actor=editor&action=resolve', { headers })).json(); + expect(filtered.total).toBe(1); + expect(filtered.events[0]).toMatchObject({ actor: 'editor@example.test', action: 'report.resolve', target: '42', detail: { batch: true } }); + const csv = await app.request('/api/admin/audit?format=csv', { headers }); + expect(csv.headers.get('content-type')).toContain('text/csv'); + expect(await csv.text()).toContain('notes.save.lyrics'); + const json = await (await app.request('/api/admin/audit?format=json', { headers })).json(); + expect(json.events).toHaveLength(2); + }); +}); diff --git a/server/db.js b/server/db.js index a3f8e4e..900001a 100644 --- a/server/db.js +++ b/server/db.js @@ -265,6 +265,17 @@ export async function initDb() { created_at INTEGER NOT NULL DEFAULT (unixepoch()), last_used_at INTEGER ); + + CREATE TABLE IF NOT EXISTS audit_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts INTEGER NOT NULL DEFAULT (unixepoch()), + actor TEXT NOT NULL, + action TEXT NOT NULL, + target TEXT NOT NULL DEFAULT '', + detail TEXT NOT NULL DEFAULT '{}' + ); + CREATE INDEX IF NOT EXISTS idx_audit_ts ON audit_log (ts DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_audit_actor_action ON audit_log (actor, action, ts DESC); `); await migrateVideoMeta(); // Uploads shared from devices (PWA share target): who sent it, and whether it diff --git a/server/deep-search.js b/server/deep-search.js index 5e387d7..493ee7f 100644 --- a/server/deep-search.js +++ b/server/deep-search.js @@ -3,6 +3,7 @@ import { db as defaultDb } from './db.js'; import * as catalog from './video-catalog.js'; import * as innertube from './innertube.js'; import * as searchCache from './search-cache.js'; +import { audit } from './audit-log.js'; const MAX_SEEDS = 200, MAX_PLAN = 500, MAX_ERRORS = 50; const clean = value => String(value || '').replace(/[\u0000-\u001f\u007f]/g, ' ').trim().slice(0, 200); @@ -191,6 +192,7 @@ export function createDeepSearchService({ db = defaultDb, searchDeep = innertube const options = { resultsPerQuery: Math.max(1, Math.min(200, Number(body.resultsPerQuery) || 50)), relatedDepth: Math.max(0, Math.min(2, Number(body.relatedDepth) || 0)), relatedTop: Math.max(1, Math.min(10, Number(body.relatedTop) || 3)), delayMs: Math.max(1000, Math.min(300000, (Number(body.delaySeconds) || 10) * 1000)), jitterMs: Math.max(0, Math.min(300000, (Number(body.jitterSeconds) || 3) * 1000)), dailyCap: Math.max(1, Math.min(10000, Number(body.dailyCap) || 1000)), channels: (Array.isArray(body.channels) ? body.channels : []).map(clean).filter(Boolean).slice(0, 25), playlists: (Array.isArray(body.playlists) ? body.playlists : []).map(clean).filter(Boolean).slice(0, 25) }; const id = randomUUID(), time = now(); await db.execute({ sql: "INSERT INTO deep_search_jobs(id,status,plan,options,state,created_at,updated_at) VALUES(?,'queued',?,?,?,?,?)", args: [id, JSON.stringify(plan), JSON.stringify(options), JSON.stringify({ queriesDone: 0, newVideos: 0, errors: [], log: [] }), time, time] }); + await audit(db, 'admin', 'deep-search.start', id, { queries: plan.length }); if (autoStart) kick(); return c.json({ ok: true, id }, 202); }); for (const action of ['pause', 'resume', 'cancel']) app.post(`/api/admin/deep-search/:id/${action}`, adminAuth, async c => { @@ -199,6 +201,7 @@ export function createDeepSearchService({ db = defaultDb, searchDeep = innertube const target = action === 'pause' ? 'paused' : action === 'resume' ? 'queued' : 'cancelled'; const marks = allowed.map(() => '?').join(','); const r = await db.execute({ sql: `UPDATE deep_search_jobs SET status=?,updated_at=? WHERE id=? AND status IN (${marks})`, args: [target, now(), id, ...allowed] }); + if (r.rowsAffected) await audit(db, 'admin', `deep-search.${action}`, id); if (action === 'resume' && r.rowsAffected && autoStart) kick(); return c.json({ ok: !!r.rowsAffected, status: r.rowsAffected ? target : undefined }, r.rowsAffected ? 200 : 409); }); diff --git a/server/notes.js b/server/notes.js index bf68373..99000e8 100644 --- a/server/notes.js +++ b/server/notes.js @@ -44,6 +44,7 @@ import { join } from 'node:path'; import { getCookie, setCookie, deleteCookie } from 'hono/cookie'; import { registerAnalyticsRoutes } from './admin-analytics.js'; import { registerTranscriptionRoutes } from './transcriptions.js'; +import { audit, registerAuditRoutes } from './audit-log.js'; export const NOTE_KINDS = new Set(['lyrics', 'chapters']); // A YouTube id or one of the server's own uploads (see uploads.js). @@ -507,6 +508,7 @@ export function registerNoteRoutes(app, deps) { if (!r.lines.length) return c.json({ ok: false, error: 'this video has no usable captions' }, 404); const data = sanitizeLyrics({ lines: r.lines.map((l) => ({ ...l, kind: 'line' })), tags: [], offset: 0 }); const saved = await db.saveNote({ videoId: id, kind: 'lyrics', data, source: 'auto', updatedBy: who.by, force: true }); + await audit(db, who.by, 'lyrics.save.auto-captions', id, { rev: saved.rev, lines: data.lines.length }); return c.json({ ok: true, rev: saved.rev, lines: data.lines.length, lang: r.lang, auto: r.auto }); } catch (err) { return c.json({ ok: false, error: err.message }, 502); @@ -551,6 +553,7 @@ export function registerNoteRoutes(app, deps) { offset: 0, }); const saved = await db.saveNote({ videoId: id, kind: 'lyrics', data, source: 'auto', updatedBy: who.by, force: true }); + await audit(db, who.by, 'lyrics.save.lrclib', id, { rev: saved.rev, lines: data.lines.length }); return c.json({ ok: true, rev: saved.rev, lines: data.lines.length, synced: hit.synced, match: { track: hit.meta.trackName, artist: hit.meta.artistName, duration: hit.meta.duration } }); } catch (err) { return c.json({ ok: false, error: err.message }, 502); @@ -578,6 +581,7 @@ export function registerNoteRoutes(app, deps) { }); if (!r.ok) return c.json({ ok: false, error: 'someone else saved a newer version', conflict: true, current: r.current }, 409); if (kind === 'lyrics') autoResolve(id, data); + await audit(db, who.by, `notes.save.${kind}`, id, { rev: r.rev, source: who.via }); return c.json({ ok: true, rev: r.rev, data }); } catch (err) { return c.json({ ok: false, error: err.message }, 500); @@ -658,11 +662,13 @@ export function registerNoteRoutes(app, deps) { const token = newApiToken(); const id = randomBytes(6).toString('hex'); await db.createApiToken({ id, label, tokenHash: hashToken(token) }); + await audit(db, 'admin', 'token.create', id, { label }); return c.json({ ok: true, id, label, token }); }); app.delete('/api/admin/tokens/:tid', requireAdmin, async (c) => { const ok = await db.deleteApiToken(c.req.param('tid')); + if (ok) await audit(db, 'admin', 'token.revoke', c.req.param('tid')); return ok ? c.json({ ok: true }) : c.json({ ok: false, error: 'not found' }, 404); }); @@ -673,6 +679,8 @@ export function registerNoteRoutes(app, deps) { await next(); }; + registerAuditRoutes(app, { db, adminAuth: requireAdminOrToken }); + if (deps.analytics) registerAnalyticsRoutes(app, { adminAuth: requireAdminOrToken, runYtdlp, ...deps.analytics }); registerTranscriptionRoutes(app, { @@ -707,7 +715,11 @@ export function registerNoteRoutes(app, deps) { app.post('/api/admin/media/:id/redownload', requireAdminOrToken, async c => { if (!media) return c.json({ ok: false, error: 'media cache unavailable' }, 503); - try { return c.json({ ok: true, ...(await media.redownload(c.req.param('id'))) }, 202); } + try { + const result = await media.redownload(c.req.param('id')); + await audit(db, 'admin', 'media.redownload', c.req.param('id'), result); + return c.json({ ok: true, ...result }, 202); + } catch (err) { return c.json({ ok: false, error: err.message }, 400); } }); @@ -727,6 +739,7 @@ export function registerNoteRoutes(app, deps) { updatedBy: `admin (rev ${old.rev})`, force: true, }); if (kind === 'lyrics') autoResolve(id, restored); + await audit(db, 'admin', `notes.restore.${kind}`, id, { fromRev: old.rev, newRev: r.rev }); return c.json({ ok: true, rev: r.rev }); }); @@ -763,11 +776,14 @@ export function registerNoteRoutes(app, deps) { try { body = await c.req.json(); } catch { /* status required below */ } if (!Number.isInteger(fid) || !['open', 'resolved'].includes(body.status)) return c.json({ ok: false, error: 'bad request' }, 400); const ok = await db.setFlagStatus(fid, body.status, 'admin'); + if (ok) await audit(db, 'admin', `report.${body.status}`, String(fid)); return ok ? c.json({ ok: true, open: await db.countOpenFlags() }) : c.json({ ok: false, error: 'not found' }, 404); }); app.delete('/api/admin/flags/:fid', requireAdmin, async (c) => { - const ok = await db.deleteFlag(Number(c.req.param('fid'))); + const id = Number(c.req.param('fid')); + const ok = await db.deleteFlag(id); + if (ok) await audit(db, 'admin', 'report.delete', String(id)); return ok ? c.json({ ok: true, open: await db.countOpenFlags() }) : c.json({ ok: false, error: 'not found' }, 404); }); diff --git a/server/package.json b/server/package.json index c144a43..248c5f4 100644 --- a/server/package.json +++ b/server/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "bun server.js", "dev": "bun --hot server.js", - "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js" + "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./audit-log.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js" }, "dependencies": { "@hono/node-server": "^1.14.0", diff --git a/tests/admin-audit.spec.js b/tests/admin-audit.spec.js new file mode 100644 index 0000000..796d02c --- /dev/null +++ b/tests/admin-audit.spec.js @@ -0,0 +1,27 @@ +const { test, expect } = require('@playwright/test'); + +test('audit tab filters, pages and links filtered exports', async ({ page }) => { + const requests = []; + await page.route('**/api/**', async route => { + const url = new URL(route.request().url()); requests.push(url.pathname + url.search); + let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { ok: true, enabled: true, admin: true }; + if (url.pathname === '/api/admin/audit') body = { + ok: true, total: 62, offset: Number(url.searchParams.get('offset')) || 0, + events: [{ id: 9, ts: 1791900000, actor: 'admin', action: 'notes.save.lyrics', target: 'dQw4w9WgXcQ', detail: { rev: 4 } }], + }; + if (url.pathname === '/api/admin/flags') body = { ok: true, open: 0, flags: [] }; + if (url.pathname === '/api/admin/media') body = { ok: true, media: [] }; + if (url.pathname === '/api/uploads') body = { ok: true, uploads: [] }; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.goto('/admin.html#audit'); + await expect(page.locator('#p-audit')).toBeVisible(); + await expect(page.locator('#auditTable')).toContainText('notes.save.lyrics'); + await page.locator('#auditActor').fill('admin'); await page.locator('#auditAction').fill('lyrics'); + await page.locator('#auditFilterForm button[type=submit]').click(); + await expect.poll(() => requests.filter(x => x.startsWith('/api/admin/audit?')).length).toBeGreaterThan(1); + await expect(page.locator('#auditJson')).toHaveAttribute('href', /actor=admin.*action=lyrics.*format=json/); + await page.locator('#auditNext').click(); + await expect(page.locator('#auditCount')).toContainText('51–'); +}); From 180a28d3e429fe09d571327b5bf3a3d043c5bc42 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 01:56:53 +0800 Subject: [PATCH 2/9] Add a paged lyrics quality queue --- frontend/admin.html | 50 ++++++++++++++++++++++++++++- playwright.admin-audit.config.js | 2 +- server/admin-tools.js | 54 ++++++++++++++++++++++++++++++++ server/admin-tools.test.js | 37 ++++++++++++++++++++++ server/package.json | 2 +- server/server.js | 3 ++ server/transcriptions.js | 2 ++ tests/admin-quality.spec.js | 26 +++++++++++++++ 8 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 server/admin-tools.js create mode 100644 server/admin-tools.test.js create mode 100644 tests/admin-quality.spec.js diff --git a/frontend/admin.html b/frontend/admin.html index 1001d3b..256d2f9 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -542,6 +542,21 @@
+
+

Lyrics quality

+

Work the songs that need attention: missing, short, machine-drafted, untimed, or reported lyrics.

+
+
+ + + +
+

+
+
50 songs per page
+
+
+

Library

@@ -733,6 +748,7 @@ const ICONS = { editor: '', reports: '', + quality: '', library: '', peers: '', history: '', @@ -741,7 +757,7 @@ audit: '', access: '', }; - const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -756,6 +772,7 @@ if (push) history.replaceState(null, '', location.pathname + location.search + '#' + name); syncBars(); if (name === 'reports') loadReports(); + if (name === 'quality') loadLyricsQuality(); if (name === 'history') loadRevs(); if (name === 'peers') loadP2p(); if (name === 'library') loadUploads(); @@ -945,6 +962,37 @@ $('auditFilterForm').addEventListener('submit', event => { event.preventDefault(); auditOffset = 0; loadAudit(); }); $('auditPrev').addEventListener('click', () => { auditOffset = Math.max(0, auditOffset - 50); loadAudit(); }); $('auditNext').addEventListener('click', () => { auditOffset += 50; loadAudit(); }); + + /* ---------------- lyrics quality queue ---------------- */ + let qualityOffset = 0; + async function loadLyricsQuality() { + const table = $('qualityTable'); table.innerHTML = 'Loading quality queue…'; + const category = $('qualityCategory').value, minLines = Number($('qualityMinLines').value) || 8; + try { + const j = await api(`/api/admin/lyrics/quality?category=${encodeURIComponent(category)}&minLines=${minLines}&limit=50&offset=${qualityOffset}`); + if (!j.ok) throw Error(j.error || 'Could not load lyrics queue.'); + $('qualityCount').textContent = `${j.total.toLocaleString()} songs · ${j.songs.length ? qualityOffset + 1 : 0}–${qualityOffset + j.songs.length}`; + $('qualityPrev').disabled = qualityOffset === 0; $('qualityNext').disabled = qualityOffset + j.songs.length >= j.total; + table.innerHTML = j.songs.length ? `SongLyricsWhy hereActions${j.songs.map(song => `${esc(song.title)}
${esc(song.channel || song.id)}${song.lines} lines${song.source === 'auto' ? ' · auto draft' : ''}${song.untimed ? ' · untimed' : ''}${esc(song.reasons.join(' · '))}${song.openReports ? ` · ${song.openReports} reports` : ''}
`).join('')}` : 'No songs match this filter. Try All issues or reduce the minimum line count.'; + } catch (error) { table.innerHTML = `${esc(error.message)}`; } + } + $('qualityFilterForm').addEventListener('submit', event => { event.preventDefault(); qualityOffset = 0; loadLyricsQuality(); }); + $('qualityPrev').addEventListener('click', () => { qualityOffset = Math.max(0, qualityOffset - 50); loadLyricsQuality(); }); + $('qualityNext').addEventListener('click', () => { qualityOffset += 50; loadLyricsQuality(); }); + $('qualityTable').addEventListener('click', async event => { + const button = event.target.closest('button'); if (!button) return; + const id = button.dataset.qualityFix || button.dataset.qualityPull || button.dataset.qualityWhisper; + if (button.dataset.qualityFix) { leOpen(id); return; } + $('qualityStatus').className = 'msg'; $('qualityStatus').textContent = button.dataset.qualityPull ? 'Looking up LRCLIB…' : 'Adding to the Whisper queue…'; + try { + const result = button.dataset.qualityPull + ? await api(`/api/notes/${encodeURIComponent(id)}/lyrics/web`, { method: 'POST', body: {} }) + : await api(`/api/admin/transcriptions/${encodeURIComponent(id)}`, { method: 'POST', body: {} }); + if (!result.ok) throw Error(result.error || 'Action failed.'); + $('qualityStatus').className = 'msg ok'; $('qualityStatus').textContent = button.dataset.qualityPull ? `LRCLIB saved revision ${result.rev}.` : `Whisper ${result.job?.status || 'job queued'}.`; + loadLyricsQuality(); + } catch (error) { $('qualityStatus').className = 'msg err'; $('qualityStatus').textContent = error.message; } + }); $('tokenForm').addEventListener('submit', async (e) => { e.preventDefault(); const label = $('tokenLabel').value.trim(); diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js index 149dd36..9f1a999 100644 --- a/playwright.admin-audit.config.js +++ b/playwright.admin-audit.config.js @@ -1,3 +1,3 @@ const { defineConfig } = require('@playwright/test'); const classic = require('./playwright.classic.config'); -module.exports = defineConfig({ ...classic, testMatch: /admin-audit\.spec\.js/ }); +module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality)\.spec\.js/ }); diff --git a/server/admin-tools.js b/server/admin-tools.js new file mode 100644 index 0000000..72e2a5d --- /dev/null +++ b/server/admin-tools.js @@ -0,0 +1,54 @@ +import { audit } from './audit-log.js'; + +const clientOf = db => typeof db?.execute === 'function' ? db : db?.db; +const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } }; +const number = value => Number(value) || 0; + +export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null }) { + const sql = clientOf(db); + + app.get('/api/admin/lyrics/quality', adminAuth, async c => { + const minLines = Math.max(1, Math.min(200, Number(c.req.query('minLines')) || 8)); + const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); + const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50)); + const category = ['all', 'missing', 'short', 'automatic', 'untimed', 'reports'].includes(c.req.query('category')) ? c.req.query('category') : 'all'; + const conditions = { + missing: 'n.video_id IS NULL', + short: 'n.video_id IS NOT NULL AND COALESCE(json_array_length(json_extract(n.data,\'$.lines\')),0) < ?', + automatic: "n.source='auto'", + untimed: "EXISTS (SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL)", + reports: "EXISTS (SELECT 1 FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open')", + }; + const parts = category === 'all' + ? [conditions.missing, conditions.short, conditions.automatic, conditions.untimed, conditions.reports] + : [conditions[category]]; + const where = `WHERE (${parts.join(' OR ')})`; + const args = category === 'short' ? [minLines] : category === 'all' ? [minLines] : []; + const query = `SELECT m.id,m.card,m.updated_at,n.data AS lyrics,n.source AS lyricSource,n.updated_at AS lyricUpdated, + COALESCE(json_array_length(json_extract(n.data,'$.lines')),0) AS lineCount, + EXISTS(SELECT 1 FROM json_each(n.data,'$.lines') line WHERE json_extract(line.value,'$.t') IS NULL) AS untimed, + (SELECT COUNT(*) FROM lyric_flags f WHERE f.video_id=m.id AND f.status='open') AS openReports + FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics' + ${where} ORDER BY openReports DESC,lyricUpdated ASC,m.updated_at DESC LIMIT ? OFFSET ?`; + const rows = await sql.execute({ sql: query, args: [...args, limit, offset] }); + const count = await sql.execute({ sql: `SELECT COUNT(*) AS total FROM video_meta m LEFT JOIN video_notes n ON n.video_id=m.id AND n.kind='lyrics' ${where}`, args }); + const songs = rows.rows.map(row => { + const card = parse(row.card); + let untimed = !!row.untimed; + let lines = number(row.lineCount); + if (row.lyrics) { + const data = parse(row.lyrics); lines ||= Array.isArray(data.lines) ? data.lines.length : 0; + if (Array.isArray(data.lines)) untimed ||= data.lines.some(line => line.t == null); + } + const reasons = []; + if (!row.lyrics) reasons.push('missing'); + else if (lines < minLines) reasons.push('short'); + if (row.lyricSource === 'auto') reasons.push('automatic'); + if (untimed) reasons.push('untimed'); + if (number(row.openReports)) reasons.push('reports'); + return { id: row.id, title: card.title || row.id, channel: card.channel || '', lines, source: row.lyricSource || null, untimed, openReports: number(row.openReports), reasons, updatedAt: number(row.lyricUpdated) }; + }); + return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' }); + }); + +} diff --git a/server/admin-tools.test.js b/server/admin-tools.test.js new file mode 100644 index 0000000..3ae7189 --- /dev/null +++ b/server/admin-tools.test.js @@ -0,0 +1,37 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Hono } from 'hono'; + +const root = mkdtempSync(join(tmpdir(), 'ytp-admin-tools-')); +process.env.DB_PATH = join(root, 'admin.db'); +const db = await import('./db.js'); +const { registerAdminToolsRoutes } = await import('./admin-tools.js'); +const app = new Hono(); +const auth = async (c, next) => c.req.header('x-admin') === 'yes' ? next() : c.json({ ok: false }, 401); + +beforeAll(async () => { + await db.initDb(); + registerAdminToolsRoutes(app, { db, adminAuth: auth }); + await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] }); + await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] }); + await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] }); +}); +afterAll(() => rmSync(root, { recursive: true, force: true })); + +describe('lyrics quality queue', () => { + test('requires admin and returns missing, untimed and short-lyrics reasons in pages', async () => { + expect((await app.request('/api/admin/lyrics/quality')).status).toBe(401); + const headers = { 'x-admin': 'yes' }; + const missing = await (await app.request('/api/admin/lyrics/quality?category=missing', { headers })).json(); + expect(missing.total).toBe(1); + expect(missing.songs[0]).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] }); + const untimed = await (await app.request('/api/admin/lyrics/quality?category=untimed', { headers })).json(); + expect(untimed.songs[0]).toMatchObject({ id: 'Untimely001', lines: 2, untimed: true }); + const short = await (await app.request('/api/admin/lyrics/quality?category=short&minLines=3', { headers })).json(); + expect(short.songs[0].reasons).toContain('short'); + const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json(); + expect(page.songs).toHaveLength(1); + }); +}); diff --git a/server/package.json b/server/package.json index 248c5f4..0973e6b 100644 --- a/server/package.json +++ b/server/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "bun server.js", "dev": "bun --hot server.js", - "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./audit-log.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js" + "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./audit-log.test.js && bun test ./admin-tools.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js" }, "dependencies": { "@hono/node-server": "^1.14.0", diff --git a/server/server.js b/server/server.js index 376ec94..096869c 100644 --- a/server/server.js +++ b/server/server.js @@ -47,6 +47,7 @@ import { initDb, upsertUser, recordVideoAccess, getUserData, createProfile, getP import { createMediaCache, HIGH, LOW, validateMedia, MediaSkip } from './media-cache.js'; import * as notesDb from './db.js'; import { registerNoteRoutes, parseLrc, sanitizeLyrics } from './notes.js'; +import { registerAdminToolsRoutes } from './admin-tools.js'; import { createRemoteHub } from './remote.js'; import { createPartyHub } from './party.js'; import { registerMediaMetaRoutes } from './media-meta.js'; @@ -2317,6 +2318,8 @@ const notes = registerNoteRoutes(app, { const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, autoStart: false, searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads, ingest: collectVideoMetadata, putSearch: searchCacheDb.put }); +registerAdminToolsRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, media, + backupDir: process.env.UPLOAD_BACKUP_DIR || pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'backups') }); // ============================================================================ // Uploads — the server's own searchable media library (see uploads.js) diff --git a/server/transcriptions.js b/server/transcriptions.js index 8c2ab3d..4f24645 100644 --- a/server/transcriptions.js +++ b/server/transcriptions.js @@ -1,6 +1,7 @@ // Explicit Whisper requests produce reviewable drafts, never published notes. import { randomUUID } from 'node:crypto'; import { db as sql } from './db.js'; +import { audit } from './audit-log.js'; const ID = /^([\w-]{11}|upl_[a-f0-9]{12})$/; const LEASE_MS = 120000; function publicJob(row) { @@ -34,6 +35,7 @@ export function registerTranscriptionRoutes(app, { db, adminAuth, workerAuth, wo SELECT ?,?,?,?,? WHERE (SELECT COUNT(*) FROM lyric_transcriptions WHERE status IN ('queued','running')) < 25`, args: [randomUUID(), id, notes.lyrics?.rev || 0, time, time] }); const row = (await sql.execute({ sql: "SELECT * FROM lyric_transcriptions WHERE video_id=? AND status IN ('queued','running')", args: [id] })).rows[0]; + if (row) await audit(db, 'admin', 'lyrics.whisper.enqueue', id, { jobId: row.id }); return row ? c.json({ ok: true, job: publicJob(row) }, 202) : c.json({ ok: false, error: 'The transcription queue is full.' }, 429); }); app.get('/api/admin/transcriptions/:video', adminAuth, async c => { diff --git a/tests/admin-quality.spec.js b/tests/admin-quality.spec.js new file mode 100644 index 0000000..8ac1e7f --- /dev/null +++ b/tests/admin-quality.spec.js @@ -0,0 +1,26 @@ +const { test, expect } = require('@playwright/test'); + +test('lyrics queue exposes issue filters and reuses LRCLIB and Whisper routes', async ({ page }) => { + const calls = []; + await page.route('**/api/**', async route => { + const req = route.request(), url = new URL(req.url()); calls.push({ path: url.pathname, method: req.method() }); + let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true }; + if (url.pathname === '/api/admin/lyrics/quality') body = { ok: true, total: 1, offset: 0, songs: [{ id: 'QualMiss001', title: 'Missing lyric sample', channel: 'Choir', lines: 0, reasons: ['missing'], openReports: 0 }] }; + if (url.pathname === '/api/notes/QualMiss001/lyrics/web') body = { ok: true, rev: 2, lines: 12 }; + if (url.pathname === '/api/admin/transcriptions/QualMiss001') body = { ok: true, job: { status: 'queued' } }; + if (url.pathname === '/api/admin/media') body = { ok: true, media: [] }; + if (url.pathname === '/api/uploads') body = { ok: true, uploads: [] }; + if (url.pathname === '/api/admin/flags') body = { ok: true, open: 0, flags: [] }; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.goto('/admin.html#quality'); + await expect(page.locator('#p-quality')).toBeVisible(); + await expect(page.locator('#qualityTable')).toContainText('Missing lyric sample'); + await page.locator('#qualityCategory').selectOption('untimed'); await page.locator('#qualityFilterForm button').click(); + await expect(page.locator('#qualityTable')).toContainText('Pull LRCLIB'); + await page.locator('[data-quality-pull]').click(); await expect(page.locator('#qualityStatus')).toContainText('revision 2'); + await page.locator('[data-quality-whisper]').click(); await expect(page.locator('#qualityStatus')).toContainText('Whisper queued'); + expect(calls.some(x => x.path.endsWith('/lyrics/web') && x.method === 'POST')).toBe(true); + expect(calls.some(x => x.path.endsWith('/transcriptions/QualMiss001') && x.method === 'POST')).toBe(true); +}); From 42c68609ec4e594ccc545fbcb82ec80287c60fb0 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 01:59:23 +0800 Subject: [PATCH 3/9] Add cache pinning and health controls --- frontend/admin.html | 53 +++++++++++++++++++++++++++++++- playwright.admin-audit.config.js | 2 +- server/admin-tools.js | 46 +++++++++++++++++++++++++++ server/admin-tools.test.js | 14 +++++++++ server/db.js | 7 +++-- server/media-cache.js | 5 +-- server/media-cache.test.js | 15 +++++++++ server/p2p-db.js | 2 +- tests/admin-cache.spec.js | 24 +++++++++++++++ 9 files changed, 161 insertions(+), 7 deletions(-) create mode 100644 tests/admin-cache.spec.js diff --git a/frontend/admin.html b/frontend/admin.html index 256d2f9..291115c 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -557,6 +557,16 @@
+
+

Cache health

+

Review server copies by state and recency. Pinned copies stay out of automatic eviction.

+
+

Server media

+

+
50 rows per page
+
+
+

Library

@@ -749,6 +759,7 @@ editor: '', reports: '', quality: '', + cache: '', library: '', peers: '', history: '', @@ -757,7 +768,7 @@ audit: '', access: '', }; - const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -773,6 +784,7 @@ syncBars(); if (name === 'reports') loadReports(); if (name === 'quality') loadLyricsQuality(); + if (name === 'cache') loadCacheHealth(); if (name === 'history') loadRevs(); if (name === 'peers') loadP2p(); if (name === 'library') loadUploads(); @@ -993,6 +1005,45 @@ loadLyricsQuality(); } catch (error) { $('qualityStatus').className = 'msg err'; $('qualityStatus').textContent = error.message; } }); + + /* ---------------- cache health and pinning ---------------- */ + let cacheOffset = 0; + async function loadCacheHealth() { + $('cacheTable').innerHTML = 'Loading cache…'; + try { + const j = await api(`/api/admin/cache/health?limit=50&offset=${cacheOffset}`); + if (!j.ok) throw Error(j.error || 'Cache health unavailable.'); + const ready = j.statuses.find(x => x.status === 'ready') || {}, pending = j.statuses.find(x => ['queued', 'downloading', 'validating'].includes(x.status)), failed = j.statuses.find(x => x.status === 'failed') || {}; + $('cacheMetrics').innerHTML = [ + ['Cache used', `${bytes(j.bytes)} / ${bytes(j.budget)}`, `${Math.round(j.utilization * 100)}% of configured budget`], + ['Ready', Number(ready.count || 0).toLocaleString(), bytes(ready.bytes || 0)], + ['Pending / failed', `${Number(pending?.count || 0)} / ${Number(failed.count || 0)}`, 'Jobs in progress or needing retry'], + ['Pinned', `${bytes(j.pinnedBytes)} / ${bytes(j.pinnedBudget)}`, j.pinnedBytes > j.pinnedBudget ? 'Over cap: unpin copies to restore automatic eviction' : 'Protected from automatic eviction'], + ].map(([label, value, detail]) => `
${esc(label)}${esc(value)}${esc(detail)}
`).join(''); + $('cacheCount').textContent = `${j.total.toLocaleString()} copies · ${j.media.length ? cacheOffset + 1 : 0}–${cacheOffset + j.media.length}`; + $('cachePrev').disabled = cacheOffset === 0; $('cacheNext').disabled = cacheOffset + j.media.length >= j.total; + $('cacheTable').innerHTML = j.media.length ? `VideoStatusSizeLast accessActions${j.media.map(item => `
Generation ${item.gen}${esc(item.status)}${item.pinned ? ' · pinned' : ''}${bytes(item.bytes)}${when(Math.floor(item.lastAccess / 1000))}
${item.status === 'ready' ? `` : ''}
`).join('')}` : 'No media copies are recorded yet.'; + } catch (error) { $('cacheStatus').className = 'msg err'; $('cacheStatus').textContent = error.message; } + } + $('cacheRefresh').addEventListener('click', loadCacheHealth); + $('cachePrev').addEventListener('click', () => { cacheOffset = Math.max(0, cacheOffset - 50); loadCacheHealth(); }); + $('cacheNext').addEventListener('click', () => { cacheOffset += 50; loadCacheHealth(); }); + $('cacheTable').addEventListener('click', async event => { + const button = event.target.closest('button'); if (!button) return; + if (button.dataset.openCache) { $('songId').value = button.dataset.openCache; showTab('editor'); return; } + let path, body = {}; + if (button.dataset.cachePin) { path = `/api/admin/cache/${encodeURIComponent(button.dataset.cachePin)}/pin`; body = { pinned: button.dataset.pinned !== 'true' }; } + else if (button.dataset.cacheEvict) { + if (!confirm('Evict this server copy now? Clients may need to stream or save it again.')) return; + path = `/api/admin/cache/${encodeURIComponent(button.dataset.cacheEvict)}/evict`; + } else if (button.dataset.cacheRedownload) { + if (!confirm('Re-download this media from its source? This replaces the server copy.')) return; + path = `/api/admin/media/${encodeURIComponent(button.dataset.cacheRedownload)}/redownload`; + } else return; + $('cacheStatus').className = 'msg'; $('cacheStatus').textContent = 'Applying change…'; + try { const result = await api(path, { method: 'POST', body }); if (!result.ok) throw Error(result.error || 'Action failed.'); $('cacheStatus').className = 'msg ok'; $('cacheStatus').textContent = 'Cache updated.'; await loadCacheHealth(); } + catch (error) { $('cacheStatus').className = 'msg err'; $('cacheStatus').textContent = error.message; } + }); $('tokenForm').addEventListener('submit', async (e) => { e.preventDefault(); const label = $('tokenLabel').value.trim(); diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js index 9f1a999..0ea8003 100644 --- a/playwright.admin-audit.config.js +++ b/playwright.admin-audit.config.js @@ -1,3 +1,3 @@ const { defineConfig } = require('@playwright/test'); const classic = require('./playwright.classic.config'); -module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality)\.spec\.js/ }); +module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache)\.spec\.js/ }); diff --git a/server/admin-tools.js b/server/admin-tools.js index 72e2a5d..f44fb9e 100644 --- a/server/admin-tools.js +++ b/server/admin-tools.js @@ -51,4 +51,50 @@ export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, bac return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' }); }); + app.get('/api/admin/cache/health', adminAuth, async c => { + const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50)); + const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); + const rows = await sql.execute({ sql: "SELECT video_id,status,size,last_access,updated_at,gen,pinned FROM media_cache ORDER BY CASE WHEN status='ready' THEN 0 ELSE 1 END,last_access ASC LIMIT ? OFFSET ?", args: [limit, offset] }); + const total = await sql.execute('SELECT COUNT(*) AS total FROM media_cache'); + const grouped = await sql.execute('SELECT status,COUNT(*) AS count,COALESCE(SUM(size),0) AS bytes,COALESCE(SUM(CASE WHEN pinned=1 THEN size ELSE 0 END),0) AS pinnedBytes FROM media_cache GROUP BY status'); + const bytes = grouped.rows.reduce((sum, row) => sum + number(row.bytes), 0); + const pinnedBytes = grouped.rows.reduce((sum, row) => sum + number(row.pinnedBytes), 0); + const budget = Math.max(0, Number(process.env.MEDIA_CACHE_MAX_BYTES) || 10 * 1024 ** 3); + const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3); + return c.json({ ok: true, budget, pinnedBudget, bytes, pinnedBytes, utilization: budget ? bytes / budget : 0, + statuses: grouped.rows.map(row => ({ status: row.status, count: number(row.count), bytes: number(row.bytes) })), + total: number(total.rows[0]?.total), limit, offset, + media: rows.rows.map(row => ({ id: row.video_id, status: row.status, bytes: number(row.size), lastAccess: number(row.last_access), updatedAt: number(row.updated_at), gen: number(row.gen), pinned: !!row.pinned })) }, 200, { 'Cache-Control': 'no-store' }); + }); + + app.post('/api/admin/cache/:id/evict', adminAuth, async c => { + const id = c.req.param('id'); + if (!/^[A-Za-z0-9_-]{6,64}$/.test(id)) return c.json({ ok: false, error: 'invalid media id' }, 400); + const row = await db.getMedia(id); + if (!row) return c.json({ ok: false, error: 'media not found' }, 404); + if (row.pinned) return c.json({ ok: false, error: 'Unpin this copy before evicting it.' }, 409); + if (!media?.evictNow) return c.json({ ok: false, error: 'media cache unavailable' }, 503); + await media.evictNow(id); + await audit(db, 'admin', 'cache.evict', id, { bytes: number(row.size) }); + return c.json({ ok: true }); + }); + + app.post('/api/admin/cache/:id/pin', adminAuth, async c => { + const id = c.req.param('id'); + let body = {}; try { body = await c.req.json(); } catch {} + if (body.pinned !== true && body.pinned !== false) return c.json({ ok: false, error: 'pinned must be boolean' }, 400); + const target = await sql.execute({ sql: "SELECT size,pinned,status FROM media_cache WHERE video_id=?", args: [id] }); + const row = target.rows[0]; + if (!row) return c.json({ ok: false, error: 'media not found' }, 404); + if (body.pinned && row.status !== 'ready') return c.json({ ok: false, error: 'Only ready cache copies can be pinned.' }, 409); + const pinnedBudget = Math.max(0, Number(process.env.MEDIA_CACHE_PINNED_MAX_BYTES) || 2 * 1024 ** 3); + if (body.pinned && !row.pinned) { + const totals = await sql.execute('SELECT COALESCE(SUM(size),0) AS bytes FROM media_cache WHERE pinned=1'); + if (number(totals.rows[0]?.bytes) + number(row.size) > pinnedBudget) return c.json({ ok: false, error: 'Pinned cache byte cap exceeded.' }, 409); + } + await sql.execute({ sql: 'UPDATE media_cache SET pinned=? WHERE video_id=?', args: [body.pinned ? 1 : 0, id] }); + await audit(db, 'admin', body.pinned ? 'cache.pin' : 'cache.unpin', id, { bytes: number(row.size) }); + return c.json({ ok: true, pinned: body.pinned }); + }); + } diff --git a/server/admin-tools.test.js b/server/admin-tools.test.js index 3ae7189..af6739a 100644 --- a/server/admin-tools.test.js +++ b/server/admin-tools.test.js @@ -6,6 +6,7 @@ import { Hono } from 'hono'; const root = mkdtempSync(join(tmpdir(), 'ytp-admin-tools-')); process.env.DB_PATH = join(root, 'admin.db'); +process.env.MEDIA_CACHE_PINNED_MAX_BYTES = '1000'; const db = await import('./db.js'); const { registerAdminToolsRoutes } = await import('./admin-tools.js'); const app = new Hono(); @@ -17,6 +18,7 @@ beforeAll(async () => { await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] }); await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] }); await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] }); + await db.db.execute({ sql: "INSERT INTO media_cache(video_id,status,size,gen) VALUES('CacheItem001','ready',700,1),('CacheItem002','ready',400,1)" }); }); afterAll(() => rmSync(root, { recursive: true, force: true })); @@ -34,4 +36,16 @@ describe('lyrics quality queue', () => { const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json(); expect(page.songs).toHaveLength(1); }); + + test('pinning is authorized and enforces the configured pinned-byte cap', async () => { + const headers = { 'x-admin': 'yes', 'content-type': 'application/json' }; + expect((await app.request('/api/admin/cache/CacheItem001/pin', { method: 'POST', headers, body: JSON.stringify({ pinned: true }) })).status).toBe(200); + const over = await app.request('/api/admin/cache/CacheItem002/pin', { method: 'POST', headers, body: JSON.stringify({ pinned: true }) }); + expect(over.status).toBe(409); + expect((await over.json()).error).toContain('byte cap'); + const refused = await app.request('/api/admin/cache/CacheItem001/evict', { method: 'POST', headers, body: '{}' }); + expect(refused.status).toBe(409); + const health = await (await app.request('/api/admin/cache/health', { headers: { 'x-admin': 'yes' } })).json(); + expect(health.pinnedBytes).toBe(700); + }); }); diff --git a/server/db.js b/server/db.js index 900001a..491973d 100644 --- a/server/db.js +++ b/server/db.js @@ -106,7 +106,8 @@ export async function initDb() { created_at INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL DEFAULT 0, last_access INTEGER NOT NULL DEFAULT 0, - hits INTEGER NOT NULL DEFAULT 0 + hits INTEGER NOT NULL DEFAULT 0, + pinned INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_media_lru @@ -278,6 +279,7 @@ export async function initDb() { CREATE INDEX IF NOT EXISTS idx_audit_actor_action ON audit_log (actor, action, ts DESC); `); await migrateVideoMeta(); + try { await db.execute('ALTER TABLE media_cache ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0'); } catch { /* already migrated */ } // Uploads shared from devices (PWA share target): who sent it, and whether it // is listed in everyone's search (admin uploads) or reachable only by link. for (const col of ['owner TEXT', 'listed INTEGER NOT NULL DEFAULT 1']) { @@ -581,6 +583,7 @@ const MEDIA_COLS = new Set([ 'status', 'gen', 'size', 'height', 'vcodec', 'acodec', 'duration', 'optimized', 'meta', 'priority', 'auto', 'attempts', 'error', 'retry_at', 'created_at', 'updated_at', 'last_access', 'hits', 'sha256', + 'pinned', ]); function rowToObj(res, row) { @@ -618,7 +621,7 @@ export async function listMedia() { // Ready rows, least-recently-played first — the eviction order. export async function listMediaLru() { const r = await db.execute( - "SELECT video_id, size, last_access FROM media_cache WHERE status = 'ready' ORDER BY last_access ASC", + "SELECT video_id, size, last_access, pinned FROM media_cache WHERE status = 'ready' ORDER BY last_access ASC", ); return r.rows.map((row) => rowToObj(r, row)); } diff --git a/server/media-cache.js b/server/media-cache.js index 14bc7cb..f92aafe 100644 --- a/server/media-cache.js +++ b/server/media-cache.js @@ -242,7 +242,7 @@ export function createMediaCache({ const order = db.listMediaEvictionOrder ? await db.listMediaEvictionOrder() : await db.listMediaLru(); for (const r of order) { if (bytes + est <= maxBytes) break; - if (r.video_id === id || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue; + if (r.video_id === id || r.pinned || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue; await evict(r.video_id, 'budget'); bytes -= r.size || 0; } @@ -735,5 +735,6 @@ export function createMediaCache({ return n; } - return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available }; + return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available, + evictNow: id => evict(id, 'admin') }; } diff --git a/server/media-cache.test.js b/server/media-cache.test.js index a422c06..7481436 100644 --- a/server/media-cache.test.js +++ b/server/media-cache.test.js @@ -271,6 +271,21 @@ describe('media cache jobs', () => { expect(await cache.getReady('lruAAAAAAA3')).not.toBeNull(); }); + test('pinned copies are skipped while automatic LRU eviction selects another row', async () => { + await clearDb(); + const size = readFileSync(fx('good.mp4')).length; + const { cache } = makeCache({ maxBytes: size * 2 + 60 * 250 * 1024 + 50_000, duration: 10 }); + await cache.init(); + await cache.ensureCached('pinAAAAAA01', { priority: HIGH }); + await cache.ensureCached('pinAAAAAA02', { priority: HIGH }); + await dbmod.upsertMedia('pinAAAAAA01', { last_access: 1000, pinned: 1 }); + await dbmod.upsertMedia('pinAAAAAA02', { last_access: 2000 }); + await cache.ensureCached('pinAAAAAA03', { priority: HIGH }); + expect((await dbmod.getMedia('pinAAAAAA01')).pinned).toBe(1); + expect(await dbmod.getMedia('pinAAAAAA02')).toBeNull(); + expect(await cache.getReady('pinAAAAAA03')).not.toBeNull(); + }); + test('eviction follows listMediaEvictionOrder when the db provides it', async () => { await clearDb(); const size = readFileSync(fx('good.mp4')).length; diff --git a/server/p2p-db.js b/server/p2p-db.js index 5f73d23..962630b 100644 --- a/server/p2p-db.js +++ b/server/p2p-db.js @@ -243,7 +243,7 @@ export async function p2pStats() { // only then the qualifying ones, least recently played first. export async function listMediaEvictionOrder({ now, keepMinViews, keepDays, keepRecentDays }) { const r = await db.execute({ - sql: `SELECT m.video_id, m.size, m.last_access, + sql: `SELECT m.video_id, m.size, m.last_access, m.pinned, COALESCE((SELECT SUM(v.n) FROM video_views v WHERE v.video_id = m.video_id AND v.day >= ?), 0) AS views FROM media_cache m WHERE m.status = 'ready'`, diff --git a/tests/admin-cache.spec.js b/tests/admin-cache.spec.js new file mode 100644 index 0000000..bfe7f84 --- /dev/null +++ b/tests/admin-cache.spec.js @@ -0,0 +1,24 @@ +const { test, expect } = require('@playwright/test'); + +test('cache actions pin, unpin, confirm eviction and request a source re-download', async ({ page }) => { + const state = { pinned: false, evicted: false, redownloaded: false }; + await page.route('**/api/**', async route => { + const req = route.request(), url = new URL(req.url()); let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true }; + else if (url.pathname === '/api/admin/cache/health') body = { ok: true, total: 1, offset: 0, budget: 1000, pinnedBudget: 900, bytes: 700, pinnedBytes: state.pinned ? 700 : 0, utilization: .7, statuses: [{ status: 'ready', count: 1, bytes: 700 }], media: [{ id: 'CacheItem001', status: 'ready', bytes: 700, lastAccess: 1791900000000, gen: 4, pinned: state.pinned }] }; + else if (url.pathname === '/api/admin/cache/CacheItem001/pin' && req.method() === 'POST') { state.pinned = req.postDataJSON().pinned; body.pinned = state.pinned; } + else if (url.pathname === '/api/admin/cache/CacheItem001/evict' && req.method() === 'POST') state.evicted = true; + else if (url.pathname === '/api/admin/media/CacheItem001/redownload' && req.method() === 'POST') state.redownloaded = true; + else if (url.pathname === '/api/admin/media') body.media = []; + else if (url.pathname === '/api/uploads') body.uploads = []; + else if (url.pathname === '/api/admin/flags') body.flags = []; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.goto('/admin.html#cache'); + await expect(page.locator('#cacheTable')).toContainText('CacheItem001'); + await page.locator('[data-cache-pin]').click(); await expect(page.locator('[data-cache-pin]')).toHaveText('Unpin'); + await page.locator('[data-cache-pin]').click(); await expect(page.locator('[data-cache-pin]')).toHaveText('Pin'); + page.once('dialog', dialog => dialog.accept()); await page.locator('[data-cache-evict]').click(); + page.once('dialog', dialog => dialog.accept()); await page.locator('[data-cache-redownload]').click(); + expect(state).toEqual({ pinned: false, evicted: true, redownloaded: true }); +}); From 34b1173ea6aa03f569e310ac4ad44503cc59daf4 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 02:02:07 +0800 Subject: [PATCH 4/9] Add searchable index health and repair tools --- frontend/admin.html | 46 +++++++++++++++++++++++++++- playwright.admin-audit.config.js | 2 +- server/admin-tools.js | 51 +++++++++++++++++++++++++++++++- server/admin-tools.test.js | 30 ++++++++++++++++--- server/server.js | 4 +-- tests/admin-index.spec.js | 22 ++++++++++++++ 6 files changed, 146 insertions(+), 9 deletions(-) create mode 100644 tests/admin-index.spec.js diff --git a/frontend/admin.html b/frontend/admin.html index 291115c..ceb629c 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -567,6 +567,15 @@
+
+

Search index

+

Coverage and discovery sources for the local search catalogue. Backfill advances in bounded, resumable pages.

+
+

Discovery sources

+

Deep search

No deep-search run recorded.

+

Catalog repair

Backfill imports older metadata in one small page. Cleanup removes only old catalogue rows with no source, cache copy, or listening history, then prunes orphaned detail records.

+
+

Library

@@ -760,6 +769,7 @@ reports: '', quality: '', cache: '', + index: '', library: '', peers: '', history: '', @@ -768,7 +778,7 @@ audit: '', access: '', }; - const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -785,6 +795,7 @@ if (name === 'reports') loadReports(); if (name === 'quality') loadLyricsQuality(); if (name === 'cache') loadCacheHealth(); + if (name === 'index') loadIndexHealth(); if (name === 'history') loadRevs(); if (name === 'peers') loadP2p(); if (name === 'library') loadUploads(); @@ -1044,6 +1055,39 @@ try { const result = await api(path, { method: 'POST', body }); if (!result.ok) throw Error(result.error || 'Action failed.'); $('cacheStatus').className = 'msg ok'; $('cacheStatus').textContent = 'Cache updated.'; await loadCacheHealth(); } catch (error) { $('cacheStatus').className = 'msg err'; $('cacheStatus').textContent = error.message; } }); + + /* ---------------- search index health ---------------- */ + async function loadIndexHealth() { + $('indexSources').innerHTML = 'Loading source coverage…'; + try { + const j = await api('/api/admin/index/health'); if (!j.ok) throw Error(j.error || 'Index health unavailable.'); + const pct = value => `${Math.round(Number(value || 0) * 100)}%`; + $('indexMetrics').innerHTML = [ + ['Indexed videos', Number(j.total).toLocaleString(), 'Searchable catalogue rows'], + ['Canonical URLs', pct(j.urlCoverage), `${Number(j.urlCount).toLocaleString()} of ${Number(j.total).toLocaleString()}`], + ['Detailed metadata', pct(j.detailCoverage), `${Number(j.detailCount).toLocaleString()} enriched records`], + ['Saved thumbnails', pct(j.thumbnailCoverage), `${Number(j.thumbnailCount).toLocaleString()} stored images`], + ].map(([label, value, detail]) => `
${esc(label)}${esc(value)}${esc(detail)}
`).join(''); + $('indexSources').innerHTML = j.sources.length ? `SourceVideosDiscoveriesLast seen${j.sources.map(x => `${esc(x.source)}${Number(x.videos).toLocaleString()}${Number(x.discoveries).toLocaleString()}${when(Math.floor(x.lastSeen / 1000))}`).join('')}` : 'No source records yet.'; + if (j.lastRun) { + $('indexLastRun').textContent = `${j.lastRun.status} · ${new Date(j.lastRun.createdAt).toLocaleString()}${j.lastRun.error ? ` · ${j.lastRun.error}` : ''}`; + $('indexRunDetail').textContent = `${Number(j.lastRun.progress.queriesDone || 0)} queries complete · ${Number(j.lastRun.progress.newVideos || 0)} new videos`; + } else { $('indexLastRun').textContent = 'No deep-search run recorded.'; $('indexRunDetail').textContent = ''; } + $('indexBackfillTable').innerHTML = j.backfill.length ? `Source tableProgressStatus${j.backfill.map(x => `${esc(x.source)}${Number(x.cursor).toLocaleString()} / ${Number(x.ceiling).toLocaleString()}${x.done ? 'Complete' : 'Ready for next page'}`).join('')}` : 'Backfill has not started. Use “Backfill next page” to initialize it.'; + } catch (error) { $('indexStatus').className = 'msg err'; $('indexStatus').textContent = error.message; } + } + $('indexRefresh').addEventListener('click', loadIndexHealth); + $('indexBackfill').addEventListener('click', async () => { + $('indexStatus').className = 'msg'; $('indexStatus').textContent = 'Backfilling one bounded page…'; + try { const j = await api('/api/admin/index/backfill', { method: 'POST', body: {} }); if (!j.ok) throw Error(j.error || 'Backfill failed.'); $('indexStatus').className = 'msg ok'; $('indexStatus').textContent = `Processed ${j.processed} records. Run again to continue.`; await loadIndexHealth(); } + catch (error) { $('indexStatus').className = 'msg err'; $('indexStatus').textContent = error.message; } + }); + $('indexCleanup').addEventListener('click', async () => { + if (!confirm('Remove only stale, unreferenced catalogue entries and orphaned details? This cannot be undone.')) return; + $('indexStatus').className = 'msg'; $('indexStatus').textContent = 'Cleaning stale entries…'; + try { const j = await api('/api/admin/index/cleanup', { method: 'POST', body: { days: 365 } }); if (!j.ok) throw Error(j.error || 'Cleanup failed.'); $('indexStatus').className = 'msg ok'; $('indexStatus').textContent = `Removed ${j.removed} stale catalogue entries.`; await loadIndexHealth(); } + catch (error) { $('indexStatus').className = 'msg err'; $('indexStatus').textContent = error.message; } + }); $('tokenForm').addEventListener('submit', async (e) => { e.preventDefault(); const label = $('tokenLabel').value.trim(); diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js index 0ea8003..9ae45ba 100644 --- a/playwright.admin-audit.config.js +++ b/playwright.admin-audit.config.js @@ -1,3 +1,3 @@ const { defineConfig } = require('@playwright/test'); const classic = require('./playwright.classic.config'); -module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache)\.spec\.js/ }); +module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index)\.spec\.js/ }); diff --git a/server/admin-tools.js b/server/admin-tools.js index f44fb9e..fae0f23 100644 --- a/server/admin-tools.js +++ b/server/admin-tools.js @@ -4,7 +4,7 @@ const clientOf = db => typeof db?.execute === 'function' ? db : db?.db; const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } }; const number = value => Number(value) || 0; -export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null }) { +export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null, backfillStep = null }) { const sql = clientOf(db); app.get('/api/admin/lyrics/quality', adminAuth, async c => { @@ -97,4 +97,53 @@ export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, bac return c.json({ ok: true, pinned: body.pinned }); }); + app.get('/api/admin/index/health', adminAuth, async c => { + const [stats, sources, latest, backfill] = await Promise.all([ + sql.execute(`SELECT COUNT(*) AS total,SUM(CASE WHEN url IS NOT NULL AND url<>'' THEN 1 ELSE 0 END) AS urls, + SUM(CASE WHEN EXISTS(SELECT 1 FROM video_details d WHERE d.video_id=video_meta.id) THEN 1 ELSE 0 END) AS details, + SUM(CASE WHEN EXISTS(SELECT 1 FROM video_thumbnails t WHERE t.video_id=video_meta.id AND t.data IS NOT NULL) THEN 1 ELSE 0 END) AS thumbnails FROM video_meta`), + sql.execute('SELECT source,COUNT(*) AS videos,SUM(discoveries) AS discoveries,MAX(last_seen) AS lastSeen FROM video_meta_sources GROUP BY source ORDER BY videos DESC'), + sql.execute('SELECT id,status,created_at,updated_at,state,error FROM deep_search_jobs ORDER BY created_at DESC LIMIT 1'), + sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source'), + ]); + const row = stats.rows[0] || {}, total = number(row.total); + const last = latest.rows[0]; + return c.json({ ok: true, total, urlCount: number(row.urls), detailCount: number(row.details), thumbnailCount: number(row.thumbnails), + urlCoverage: total ? number(row.urls) / total : 0, detailCoverage: total ? number(row.details) / total : 0, thumbnailCoverage: total ? number(row.thumbnails) / total : 0, + sources: sources.rows.map(x => ({ source: x.source, videos: number(x.videos), discoveries: number(x.discoveries), lastSeen: number(x.lastSeen) })), + lastRun: last ? { id: last.id, status: last.status, createdAt: number(last.created_at), updatedAt: number(last.updated_at), error: last.error || null, progress: parse(last.state) } : null, + backfill: backfill.rows.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) }, 200, { 'Cache-Control': 'no-store' }); + }); + + app.post('/api/admin/index/backfill', adminAuth, async c => { + if (!backfillStep) return c.json({ ok: false, error: 'Catalog backfill is unavailable.' }, 503); + const processed = await backfillStep(100); + const state = (await sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source')).rows; + await audit(db, 'admin', 'index.backfill', '', { processed }); + return c.json({ ok: true, processed, backfill: state.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) }); + }); + + app.post('/api/admin/index/cleanup', adminAuth, async c => { + let body = {}; try { body = await c.req.json(); } catch {} + const days = Math.max(180, Math.min(3650, Number(body.days) || 365)); + const cutoff = Date.now() - days * 86400_000; + const candidates = await sql.execute({ sql: `SELECT COUNT(*) AS n FROM video_meta m WHERE m.updated_at c.req.header('x-admin') === 'yes' ? next() : c.j beforeAll(async () => { await db.initDb(); - registerAdminToolsRoutes(app, { db, adminAuth: auth }); + registerAdminToolsRoutes(app, { db, adminAuth: auth, backfillStep: async () => 3 }); await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] }); await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] }); await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] }); await db.db.execute({ sql: "INSERT INTO media_cache(video_id,status,size,gen) VALUES('CacheItem001','ready',700,1),('CacheItem002','ready',400,1)" }); + await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at,url) VALUES(?,?,?,?,?,?)', args: ['IndexSong001', JSON.stringify({ title: 'Indexed song' }), 'indexed song', 1, Date.now(), 'https://www.youtube.com/watch?v=IndexSong001'] }); + await db.db.execute({ sql: "INSERT INTO video_meta_sources(video_id,source,last_seen) VALUES('IndexSong001','deep',?)", args: [Date.now()] }); + await db.db.execute({ sql: "INSERT INTO video_details(video_id,metadata,updated_at) VALUES('IndexSong001','{}',?)", args: [Date.now()] }); + await db.db.execute({ sql: "INSERT INTO video_thumbnails(video_id,url,data,mime,size,fetched_at) VALUES('IndexSong001','https://i.ytimg.com/vi/IndexSong001/hqdefault.jpg',x'01','image/jpeg',1,?)", args: [Date.now()] }); + await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,1)', args: ['OldOrphan01', JSON.stringify({ title: 'Old orphan' }), 'old orphan', 1] }); + await db.db.execute({ sql: "INSERT INTO deep_search_jobs(id,status,plan,options,state,created_at,updated_at) VALUES('job-1','complete','[]','{}','{\"queriesDone\":2,\"newVideos\":5}',1,2)" }); }); afterAll(() => rmSync(root, { recursive: true, force: true })); @@ -27,12 +33,12 @@ describe('lyrics quality queue', () => { expect((await app.request('/api/admin/lyrics/quality')).status).toBe(401); const headers = { 'x-admin': 'yes' }; const missing = await (await app.request('/api/admin/lyrics/quality?category=missing', { headers })).json(); - expect(missing.total).toBe(1); - expect(missing.songs[0]).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] }); + expect(missing.total).toBeGreaterThanOrEqual(1); + expect(missing.songs.find(song => song.id === 'QualMiss001')).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] }); const untimed = await (await app.request('/api/admin/lyrics/quality?category=untimed', { headers })).json(); expect(untimed.songs[0]).toMatchObject({ id: 'Untimely001', lines: 2, untimed: true }); const short = await (await app.request('/api/admin/lyrics/quality?category=short&minLines=3', { headers })).json(); - expect(short.songs[0].reasons).toContain('short'); + expect(short.songs.find(song => song.id === 'Untimely001').reasons).toContain('short'); const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json(); expect(page.songs).toHaveLength(1); }); @@ -48,4 +54,20 @@ describe('lyrics quality queue', () => { const health = await (await app.request('/api/admin/cache/health', { headers: { 'x-admin': 'yes' } })).json(); expect(health.pinnedBytes).toBe(700); }); + + test('index health reports coverage, source and deep-search data; repair actions are audited', async () => { + const headers = { 'x-admin': 'yes', 'content-type': 'application/json' }; + const health = await (await app.request('/api/admin/index/health', { headers })).json(); + expect(health.total).toBeGreaterThanOrEqual(1); + expect(health.urlCount).toBeGreaterThanOrEqual(1); + expect(health.detailCount).toBe(1); + expect(health.thumbnailCount).toBe(1); + expect(health.sources[0].source).toBe('deep'); + expect(health.lastRun.progress.queriesDone).toBe(2); + const step = await (await app.request('/api/admin/index/backfill', { method: 'POST', headers, body: '{}' })).json(); + expect(step.processed).toBe(3); + const clean = await (await app.request('/api/admin/index/cleanup', { method: 'POST', headers, body: JSON.stringify({ days: 180 }) })).json(); + expect(clean.removed).toBeGreaterThanOrEqual(1); + expect((await db.db.execute({ sql: "SELECT COUNT(*) AS n FROM video_meta WHERE id='OldOrphan01'" })).rows[0].n).toBe(0); + }); }); diff --git a/server/server.js b/server/server.js index 096869c..7e6d3ac 100644 --- a/server/server.js +++ b/server/server.js @@ -64,7 +64,7 @@ import { sha256Range } from './hash.js'; import * as innertube from './innertube.js'; import { createRelatedService, registerRelatedRoute } from './related.js'; import * as searchCacheDb from './search-cache.js'; -import { ingest as collectVideoMetadata, syncListening, linkListening, startThumbnails } from './video-catalog.js'; +import { ingest as collectVideoMetadata, syncListening, linkListening, startThumbnails, backfillStep } from './video-catalog.js'; import { registerCatalogRoutes } from './recommendations.js'; import QRCode from 'qrcode'; import { createYtdlpPool } from './ytdlp-pool.js'; @@ -2319,7 +2319,7 @@ const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads, ingest: collectVideoMetadata, putSearch: searchCacheDb.put }); registerAdminToolsRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, media, - backupDir: process.env.UPLOAD_BACKUP_DIR || pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'backups') }); + backupDir: process.env.UPLOAD_BACKUP_DIR || pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'backups'), backfillStep }); // ============================================================================ // Uploads — the server's own searchable media library (see uploads.js) diff --git a/tests/admin-index.spec.js b/tests/admin-index.spec.js new file mode 100644 index 0000000..cfcb6b0 --- /dev/null +++ b/tests/admin-index.spec.js @@ -0,0 +1,22 @@ +const { test, expect } = require('@playwright/test'); + +test('search index tab shows source coverage and runs bounded repair actions', async ({ page }) => { + const actions = []; + await page.route('**/api/**', async route => { + const req = route.request(), url = new URL(req.url()); let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true }; + else if (url.pathname === '/api/admin/index/health') body = { ok: true, total: 100, urlCount: 90, detailCount: 40, thumbnailCount: 75, urlCoverage: .9, detailCoverage: .4, thumbnailCoverage: .75, sources: [{ source: 'deep', videos: 70, discoveries: 300, lastSeen: 1791900000000 }], lastRun: { status: 'complete', createdAt: 1791900000000, progress: { queriesDone: 20, newVideos: 400 } }, backfill: [{ source: 'media_cache', cursor: 10, ceiling: 20, done: false }] }; + else if (url.pathname === '/api/admin/index/backfill' && req.method() === 'POST') { actions.push('backfill'); body.processed = 3; } + else if (url.pathname === '/api/admin/index/cleanup' && req.method() === 'POST') { actions.push('cleanup'); body.removed = 2; } + else if (url.pathname === '/api/admin/media') body.media = []; + else if (url.pathname === '/api/uploads') body.uploads = []; + else if (url.pathname === '/api/admin/flags') body.flags = []; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.goto('/admin.html#index'); + await expect(page.locator('#indexMetrics')).toContainText('90%'); + await expect(page.locator('#indexSources')).toContainText('deep'); + await page.locator('#indexBackfill').click(); await expect(page.locator('#indexStatus')).toContainText('Processed 3'); + page.once('dialog', dialog => dialog.accept()); await page.locator('#indexCleanup').click(); await expect(page.locator('#indexStatus')).toContainText('Removed 2'); + expect(actions).toEqual(['backfill', 'cleanup']); +}); From efac16ddaaf835e58a673def6678430d5c30b6df Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 02:04:02 +0800 Subject: [PATCH 5/9] Add song-level report triage --- frontend/admin.html | 12 +++++++++++- playwright.admin-audit.config.js | 2 +- server/db.js | 10 +++++++--- server/notes.js | 16 ++++++++++++++-- server/notes.test.js | 15 +++++++++++++++ tests/admin-reports.spec.js | 25 +++++++++++++++++++++++++ 6 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 tests/admin-reports.spec.js diff --git a/frontend/admin.html b/frontend/admin.html index ceb629c..268dd62 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -532,6 +532,7 @@

Reported lines

Listeners flag lines that look wrong. Fix one in the editor and its report closes itself; or mark it resolved if it was fine.

+
@@ -1320,7 +1321,8 @@ async function loadReports() { const box = $('repList'); box.innerHTML = '
Loading…
'; - const j = await api('/api/admin/flags?status=' + repStatus); + const age = Number($('reportAge')?.value || 0); + const j = await api(`/api/admin/flags?status=${repStatus}&olderThanDays=${age}`); if (!j.ok) { box.innerHTML = `
${esc(j.error || 'failed')}
`; return; } setBadge(j.open); if (!j.flags.length) { @@ -1341,6 +1343,7 @@

${esc(s.title || vid)}

${esc(s.channel || '')} ${esc(vid)}
+ ${repStatus === 'open' ? `` : ''}
${[...s.lines].map(([text, fl]) => `
@@ -1362,6 +1365,12 @@ $('repList').addEventListener('click', async (e) => { const b = e.target.closest('button'); if (!b) return; if (b.dataset.fix) { leOpen(b.dataset.fix, b.dataset.text); return; } + if (b.dataset.resolveSong) { + if (!confirm(`Resolve every open report for ${b.dataset.resolveSong}?`)) return; + const result = await api(`/api/admin/flags/song/${encodeURIComponent(b.dataset.resolveSong)}/resolve`, { method: 'POST', body: {} }); + if (!result.ok) alert(result.error || 'Could not resolve reports'); + await loadReports(); leReloadFlagsOnly(); return; + } const each = async (ids, fn) => { for (const id of ids.split(',')) await fn(id); await loadReports(); leReloadFlagsOnly(); }; if (b.dataset.resall) each(b.dataset.resall, (id) => api(`/api/admin/flags/${id}`, { method: 'POST', body: { status: 'resolved' } })); else if (b.dataset.reopen) each(b.dataset.reopen, (id) => api(`/api/admin/flags/${id}`, { method: 'POST', body: { status: 'open' } })); @@ -1375,6 +1384,7 @@ } $('fOpen').addEventListener('click', () => setFilter('open')); $('fRes').addEventListener('click', () => setFilter('resolved')); + $('reportAge').addEventListener('change', loadReports); $('repRefresh').addEventListener('click', loadReports); /* ======================================================================== diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js index 9ae45ba..a537d91 100644 --- a/playwright.admin-audit.config.js +++ b/playwright.admin-audit.config.js @@ -1,3 +1,3 @@ const { defineConfig } = require('@playwright/test'); const classic = require('./playwright.classic.config'); -module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index)\.spec\.js/ }); +module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index|reports)\.spec\.js/ }); diff --git a/server/db.js b/server/db.js index 491973d..ddc6782 100644 --- a/server/db.js +++ b/server/db.js @@ -493,11 +493,15 @@ export async function listFlagsForVideo(videoId, status = 'open') { } // Newest first across every video; status 'all' returns both. -export async function listAllFlags({ status = 'open', limit = 200 } = {}) { - const where = status === 'all' ? '' : 'WHERE status = ?'; +export async function listAllFlags({ status = 'open', limit = 200, olderThan = 0 } = {}) { + const filters = []; + const args = []; + if (status !== 'all') { filters.push('status = ?'); args.push(status); } + if (Number(olderThan) > 0) { filters.push('created_at <= ?'); args.push(Math.floor(Number(olderThan))); } + const where = filters.length ? `WHERE ${filters.join(' AND ')}` : ''; const r = await db.execute({ sql: `SELECT * FROM lyric_flags ${where} ORDER BY created_at DESC, id DESC LIMIT ?`, - args: status === 'all' ? [limit] : [status, limit], + args: [...args, limit], }); return r.rows.map(flagRow); } diff --git a/server/notes.js b/server/notes.js index 99000e8..c89f895 100644 --- a/server/notes.js +++ b/server/notes.js @@ -762,12 +762,24 @@ export function registerNoteRoutes(app, deps) { app.get('/api/admin/flags', requireAdminOrToken, async (c) => { const status = ['open', 'resolved', 'all'].includes(c.req.query('status')) ? c.req.query('status') : 'open'; + const olderThanDays = Math.max(0, Math.min(3650, Number(c.req.query('olderThanDays')) || 0)); + const olderThan = olderThanDays ? Math.floor(Date.now() / 1000) - olderThanDays * 86400 : 0; const cache = new Map(); const flags = []; - for (const f of await db.listAllFlags({ status, limit: 300 })) { + for (const f of await db.listAllFlags({ status, limit: 300, olderThan })) { flags.push({ ...f, ...(await songLabel(f.videoId, cache)) }); } - return c.json({ ok: true, open: await db.countOpenFlags(), flags }); + return c.json({ ok: true, open: await db.countOpenFlags(), flags, olderThanDays }); + }); + + app.post('/api/admin/flags/song/:videoId/resolve', requireAdmin, async (c) => { + const videoId = c.req.param('videoId'); + if (badId(videoId)) return c.json({ ok: false, error: 'invalid video id' }, 400); + const flags = await db.listFlagsForVideo(videoId, 'open'); + let resolved = 0; + for (const flag of flags) if (await db.setFlagStatus(flag.id, 'resolved', 'admin')) resolved++; + if (resolved) await audit(db, 'admin', 'report.song.resolve', videoId, { resolved }); + return c.json({ ok: true, resolved, open: await db.countOpenFlags() }); }); app.post('/api/admin/flags/:fid', requireAdmin, async (c) => { diff --git a/server/notes.test.js b/server/notes.test.js index 65a9d7a..0f86468 100644 --- a/server/notes.test.js +++ b/server/notes.test.js @@ -284,3 +284,18 @@ test('admin song list includes metadata-only phone songs for lyrics lookup', asy expect((await app.request(`/api/admin/media/${id}/redownload`, { method: 'POST', headers: { Cookie: cookie } })).status).toBe(202); expect((await app.request('/api/admin/media/phoneSong01/redownload', { method: 'POST' })).status).toBe(401); }); + +test('report triage filters aging flags and resolves every open report for one song', async () => { + const id = VID; + const a = await dbmod.upsertFlag({ videoId: id, index: 0, text: 'Flagged line', rev: 1, reason: 'words', note: '', reporter: 'listener-a' }); + const b = await dbmod.upsertFlag({ videoId: id, index: 0, text: 'Flagged line', rev: 1, reason: 'typo', note: '', reporter: 'listener-b' }); + await dbmod.db.execute({ sql: 'UPDATE lyric_flags SET created_at=? WHERE id IN (?,?)', args: [Math.floor(Date.now() / 1000) - 10 * 86400, a.id, b.id] }); + const cookie = await adminCookie(); + const aged = await (await app.request('/api/admin/flags?status=open&olderThanDays=7', { headers: { Cookie: cookie } })).json(); + expect(aged.flags.filter(flag => flag.videoId === id)).toHaveLength(2); + const unauthorized = await app.request(`/api/admin/flags/song/${id}/resolve`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}' }); + expect(unauthorized.status).toBe(401); + const response = await app.request(`/api/admin/flags/song/${id}/resolve`, json('POST', {}, { Cookie: cookie })); + expect(await response.json()).toMatchObject({ ok: true, resolved: 2 }); + expect((await dbmod.listFlagsForVideo(id, 'open'))).toHaveLength(0); +}); diff --git a/tests/admin-reports.spec.js b/tests/admin-reports.spec.js new file mode 100644 index 0000000..8c8ed76 --- /dev/null +++ b/tests/admin-reports.spec.js @@ -0,0 +1,25 @@ +const { test, expect } = require('@playwright/test'); + +test('report triage filters age and resolves a song batch with confirmation', async ({ page }) => { + const requests = []; + await page.route('**/api/**', async route => { + const request = route.request(); const url = new URL(request.url()); requests.push({ path: url.pathname, search: url.search, method: request.method() }); + let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { ok: true, enabled: true, admin: true }; + if (url.pathname === '/api/admin/flags' && request.method() === 'GET') body = { ok: true, open: 2, flags: [ + { id: 1, videoId: 'dQw4w9WgXcQ', title: 'Sample song', channel: 'Choir', text: 'Flagged line', reporter: 'Listener', reason: 'words', status: 'open', createdAt: 1791300000, note: 'Wrong lyric' }, + { id: 2, videoId: 'dQw4w9WgXcQ', title: 'Sample song', channel: 'Choir', text: 'Flagged line', reporter: 'Second listener', reason: 'typo', status: 'open', createdAt: 1791300000, note: '' }, + ] }; + if (url.pathname.endsWith('/resolve') && request.method() === 'POST') body = { ok: true, resolved: 2, open: 0 }; + if (url.pathname === '/api/notes/dQw4w9WgXcQ') body = { ok: true, lyrics: null, chapters: null }; + if (url.pathname === '/api/admin/notes/recent') body = { ok: true, notes: [] }; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + page.on('dialog', dialog => dialog.accept()); + await page.goto('/admin.html#reports'); + await expect(page.locator('#repList')).toContainText('Second listener'); + await page.locator('#reportAge').selectOption('30'); + await expect.poll(() => requests.some(r => r.path === '/api/admin/flags' && r.search.includes('olderThanDays=30'))).toBe(true); + await page.locator('[data-resolve-song="dQw4w9WgXcQ"]').click(); + await expect.poll(() => requests.some(r => r.path === '/api/admin/flags/song/dQw4w9WgXcQ/resolve' && r.method === 'POST')).toBe(true); +}); From e3821621f39a7345d0b18b40e7f3c066b3b99359 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 02:06:33 +0800 Subject: [PATCH 6/9] Add notes backup management --- frontend/admin.html | 50 ++++++++++++++- playwright.admin-audit.config.js | 2 +- server/admin-tools.js | 105 +++++++++++++++++++++++++++++++ server/admin-tools.test.js | 25 +++++++- tests/admin-backups.spec.js | 25 ++++++++ 5 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 tests/admin-backups.spec.js diff --git a/frontend/admin.html b/frontend/admin.html index 268dd62..11a1634 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -646,6 +646,15 @@
+
+

Backups

+

Daily note snapshots can be inspected, downloaded, and restored. Every restore writes a separate safety snapshot first.

+

Notes snapshots

Export SQLite dump
+

+
+

Preview and restore

+
+

Server analytics

@@ -774,12 +783,13 @@ library: '', peers: '', history: '', + backups: '', analytics: '', 'deep-search': '', audit: '', access: '', }; - const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -798,6 +808,7 @@ if (name === 'cache') loadCacheHealth(); if (name === 'index') loadIndexHealth(); if (name === 'history') loadRevs(); + if (name === 'backups') loadBackups(); if (name === 'peers') loadP2p(); if (name === 'library') loadUploads(); if (name === 'access') loadTokens(); @@ -987,6 +998,43 @@ $('auditPrev').addEventListener('click', () => { auditOffset = Math.max(0, auditOffset - 50); loadAudit(); }); $('auditNext').addEventListener('click', () => { auditOffset += 50; loadAudit(); }); + /* ---------------- note backups ---------------- */ + async function loadBackups() { + const table = $('backupTable'); table.innerHTML = 'Loading backups…'; + try { + const result = await api('/api/admin/backups'); if (!result.ok) throw Error(result.error || 'Backup list unavailable.'); + const valid = result.backups.filter(item => item.valid); + $('backupDate').innerHTML = valid.map(item => ``).join(''); + table.innerHTML = result.backups.length ? `DateSizeNotesIntegrity${result.backups.map(item => `${esc(item.date)}${bytes(item.bytes)}${item.notes}${item.valid ? 'Verified' : 'Invalid JSON'}${item.valid ? `Download` : ''}`).join('')}` : 'No daily backups found yet.'; + $('backupPreview').disabled = $('backupRestore').disabled = !valid.length; + if (valid.length) await previewBackup(); + } catch (error) { table.innerHTML = `${esc(error.message)}`; } + } + async function previewBackup() { + const date = $('backupDate').value; if (!date) return; + const result = await api(`/api/admin/backups/${encodeURIComponent(date)}`); + if (!result.ok) { $('backupDiff').textContent = result.error || 'Could not preview backup.'; return; } + const diff = result.diff; + $('backupDiff').innerHTML = `

${esc(date)} · ${diff.changed} changed · ${diff.added} only in backup · ${diff.removed} only live

    ${diff.preview.map(item => `
  • ${esc(item.key)} — ${esc(item.change)}
    • `).join('') || '
  • No differences
    • '}
`; + } + $('backupNow').addEventListener('click', async () => { + $('backupStatus').textContent = 'Writing snapshot…'; + const result = await api('/api/admin/backups', { method: 'POST', body: {} }); + $('backupStatus').className = `msg ${result.ok ? 'ok' : 'err'}`; $('backupStatus').textContent = result.ok ? `Saved ${result.notes} notes to ${result.date}.` : result.error || 'Backup failed.'; + await loadBackups(); + }); + $('backupDate').addEventListener('change', previewBackup); $('backupPreview').addEventListener('click', previewBackup); + $('backupTable').addEventListener('click', event => { const button = event.target.closest('[data-backup-preview]'); if (button) { $('backupDate').value = button.dataset.backupPreview; previewBackup(); } }); + $('backupRestore').addEventListener('click', async () => { + const date = $('backupDate').value, videoId = $('backupSong').value.trim(); if (!date) return; + const scope = videoId ? `song ${videoId}` : 'all shared notes'; + if (!confirm(`Restore ${scope} from ${date}? A safety snapshot will be created first.`)) return; + $('backupStatus').textContent = 'Creating safety snapshot and restoring…'; + const result = await api(`/api/admin/backups/${encodeURIComponent(date)}/restore`, { method: 'POST', body: videoId ? { videoId } : {} }); + $('backupStatus').className = `msg ${result.ok ? 'ok' : 'err'}`; $('backupStatus').textContent = result.ok ? `Restored ${result.restored} note documents. Safety snapshot: ${result.safetySnapshot}` : result.error || 'Restore failed.'; + await loadBackups(); + }); + /* ---------------- lyrics quality queue ---------------- */ let qualityOffset = 0; async function loadLyricsQuality() { diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js index a537d91..bcdccd7 100644 --- a/playwright.admin-audit.config.js +++ b/playwright.admin-audit.config.js @@ -1,3 +1,3 @@ const { defineConfig } = require('@playwright/test'); const classic = require('./playwright.classic.config'); -module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index|reports)\.spec\.js/ }); +module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index|reports|backups)\.spec\.js/ }); diff --git a/server/admin-tools.js b/server/admin-tools.js index fae0f23..d8116a8 100644 --- a/server/admin-tools.js +++ b/server/admin-tools.js @@ -1,4 +1,6 @@ import { audit } from './audit-log.js'; +import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; const clientOf = db => typeof db?.execute === 'function' ? db : db?.db; const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } }; @@ -7,6 +9,109 @@ const number = value => Number(value) || 0; export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null, backfillStep = null }) { const sql = clientOf(db); + const backupFile = date => join(backupDir || '.', `notes-${date}.json`); + const backupDate = value => /^\d{4}-\d{2}-\d{2}$/.test(value || '') && !Number.isNaN(Date.parse(`${value}T00:00:00Z`)); + async function readBackup(date) { + if (!backupDate(date)) return null; + try { + const data = JSON.parse(await readFile(backupFile(date), 'utf8')); + if (!Array.isArray(data.notes) || data.notes.some(note => !note?.videoId || !['lyrics', 'chapters'].includes(note.kind) || !note.data)) return null; + return data; + } catch { return null; } + } + async function createBackup({ safety = false } = {}) { + if (!backupDir || typeof db.allNotes !== 'function') throw new Error('Backup storage unavailable'); + await mkdir(safety ? join(backupDir, 'snapshots') : backupDir, { recursive: true }); + const now = new Date(); + const date = now.toISOString().slice(0, 10); + const file = safety ? join(backupDir, 'snapshots', `notes-${date}-${now.toISOString().slice(11, 19).replaceAll(':', '-')}-safety.json`) : backupFile(date); + const payload = { exportedAt: now.toISOString(), notes: await db.allNotes() }; + await writeFile(file, JSON.stringify(payload)); + return { file, date, notes: payload.notes.length }; + } + + app.get('/api/admin/backups', adminAuth, async c => { + if (!backupDir) return c.json({ ok: false, error: 'Backup storage unavailable' }, 503); + let names = []; try { names = await readdir(backupDir); } catch {} + const backups = []; + for (const name of names.filter(value => /^notes-\d{4}-\d{2}-\d{2}\.json$/.test(value)).sort().reverse()) { + const date = name.slice(6, 16); const data = await readBackup(date); const info = await stat(backupFile(date)).catch(() => null); + backups.push({ date, bytes: info?.size || 0, exportedAt: data?.exportedAt || null, notes: data?.notes.length || 0, valid: !!data }); + } + return c.json({ ok: true, backups }, 200, { 'Cache-Control': 'no-store' }); + }); + + app.get('/api/admin/backups/:date', adminAuth, async c => { + const data = await readBackup(c.req.param('date')); + if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404); + const live = await db.allNotes(); + const index = rows => new Map(rows.map(row => [`${row.videoId}:${row.kind}`, JSON.stringify(row.data)])); + const old = index(data.notes), current = index(live); + const added = [...old.keys()].filter(key => !current.has(key)); + const removed = [...current.keys()].filter(key => !old.has(key)); + const changed = [...old.keys()].filter(key => current.has(key) && old.get(key) !== current.get(key)); + return c.json({ ok: true, date: c.req.param('date'), exportedAt: data.exportedAt, total: data.notes.length, + diff: { added: added.length, removed: removed.length, changed: changed.length, preview: [...changed.slice(0, 30).map(key => ({ key, change: 'changed' })), ...added.slice(0, 15).map(key => ({ key, change: 'only in backup' })), ...removed.slice(0, 15).map(key => ({ key, change: 'only live' }))] } }); + }); + + app.get('/api/admin/backups/:date/download', adminAuth, async c => { + const data = await readBackup(c.req.param('date')); + if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404); + c.header('Content-Disposition', `attachment; filename="notes-${c.req.param('date')}.json"`); + return c.json(data); + }); + + app.post('/api/admin/backups', adminAuth, async c => { + try { + const result = await createBackup(); + await audit(db, 'admin', 'backup.create', result.date, { notes: result.notes }); + return c.json({ ok: true, ...result }); + } catch (error) { return c.json({ ok: false, error: error.message }, 503); } + }); + + app.post('/api/admin/backups/:date/restore', adminAuth, async c => { + const date = c.req.param('date'); const data = await readBackup(date); + if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404); + let body = {}; try { body = await c.req.json(); } catch {} + const selected = body.videoId ? data.notes.filter(note => note.videoId === body.videoId) : data.notes; + if (body.videoId && !selected.length) return c.json({ ok: false, error: 'Song is not present in this backup' }, 404); + const safety = await createBackup({ safety: true }); + for (const note of selected) await db.saveNote({ videoId: note.videoId, kind: note.kind, data: note.data, source: 'restore', updatedBy: `admin (backup ${date})`, force: true }); + if (!body.videoId) { + const keep = new Set(data.notes.map(note => `${note.videoId}:${note.kind}`)); + for (const note of await db.allNotes()) if (!keep.has(`${note.videoId}:${note.kind}`)) { + await sql.execute({ sql: 'DELETE FROM video_notes WHERE video_id=? AND kind=?', args: [note.videoId, note.kind] }); + } + } + await audit(db, 'admin', 'backup.restore', date, { videoId: body.videoId || null, restored: selected.length, safetySnapshot: safety.file }); + return c.json({ ok: true, restored: selected.length, safetySnapshot: safety.file }); + }); + + app.get('/api/admin/backups/export/sqlite-dump', adminAuth, async c => { + const schema = await sql.execute("SELECT type,name,sql FROM sqlite_master WHERE type IN ('table','index','trigger') AND name NOT LIKE 'sqlite_%' AND sql IS NOT NULL ORDER BY CASE type WHEN 'table' THEN 0 ELSE 1 END,name"); + const quote = value => { + if (value == null) return 'NULL'; + if (typeof value === 'number') return Number.isFinite(value) ? String(value) : 'NULL'; + if (typeof value === 'bigint') return String(value); + if (value instanceof Uint8Array) return `X'${Buffer.from(value).toString('hex')}'`; + return `'${String(value).replaceAll("'", "''")}'`; + }; + const statements = ['PRAGMA foreign_keys=OFF;', 'BEGIN TRANSACTION;']; + for (const table of schema.rows.filter(row => row.type === 'table')) { + statements.push(`${table.sql};`); + const rows = await sql.execute(`SELECT * FROM "${table.name.replaceAll('"', '""')}"`); + for (const row of rows.rows) { + const columns = Object.keys(row); + if (!columns.length) continue; + statements.push(`INSERT INTO "${table.name}" (${columns.map(name => `"${name.replaceAll('"', '""')}"`).join(',')}) VALUES (${columns.map(name => quote(row[name])).join(',')});`); + } + } + for (const object of schema.rows.filter(row => row.type !== 'table')) statements.push(`${object.sql};`); + statements.push('COMMIT;'); + c.header('Content-Disposition', 'attachment; filename="ytplayer-notes-dump.sql"'); + return c.text(statements.join('\n'), 200, { 'Content-Type': 'application/sql; charset=utf-8' }); + }); + app.get('/api/admin/lyrics/quality', adminAuth, async c => { const minLines = Math.max(1, Math.min(200, Number(c.req.query('minLines')) || 8)); const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); diff --git a/server/admin-tools.test.js b/server/admin-tools.test.js index d85d21f..6643d32 100644 --- a/server/admin-tools.test.js +++ b/server/admin-tools.test.js @@ -1,5 +1,6 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { mkdtempSync, rmSync } from 'node:fs'; +import { existsSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Hono } from 'hono'; @@ -10,11 +11,12 @@ process.env.MEDIA_CACHE_PINNED_MAX_BYTES = '1000'; const db = await import('./db.js'); const { registerAdminToolsRoutes } = await import('./admin-tools.js'); const app = new Hono(); +const backupDir = join(root, 'backups'); const auth = async (c, next) => c.req.header('x-admin') === 'yes' ? next() : c.json({ ok: false }, 401); beforeAll(async () => { await db.initDb(); - registerAdminToolsRoutes(app, { db, adminAuth: auth, backfillStep: async () => 3 }); + registerAdminToolsRoutes(app, { db, adminAuth: auth, backupDir, backfillStep: async () => 3 }); await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] }); await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] }); await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] }); @@ -70,4 +72,25 @@ describe('lyrics quality queue', () => { expect(clean.removed).toBeGreaterThanOrEqual(1); expect((await db.db.execute({ sql: "SELECT COUNT(*) AS n FROM video_meta WHERE id='OldOrphan01'" })).rows[0].n).toBe(0); }); + + test('backup list verifies, previews, downloads, safely restores and exports a SQLite dump', async () => { + const headers = { 'x-admin': 'yes', 'content-type': 'application/json' }; + await db.saveNote({ videoId: 'BackupSong01', kind: 'lyrics', data: { lines: [{ t: 1, text: 'Before' }] }, force: true, source: 'user' }); + const created = await (await app.request('/api/admin/backups', { method: 'POST', headers, body: '{}' })).json(); + expect(created.ok).toBe(true); + await db.saveNote({ videoId: 'BackupSong01', kind: 'lyrics', data: { lines: [{ t: 1, text: 'After' }] }, force: true, source: 'user' }); + const list = await (await app.request('/api/admin/backups', { headers })).json(); + expect(list.backups.find(item => item.date === created.date)).toMatchObject({ valid: true, notes: created.notes }); + const preview = await (await app.request(`/api/admin/backups/${created.date}`, { headers })).json(); + expect(preview.diff.changed).toBe(1); + const download = await app.request(`/api/admin/backups/${created.date}/download`, { headers }); + expect(download.status).toBe(200); + const restored = await (await app.request(`/api/admin/backups/${created.date}/restore`, { method: 'POST', headers, body: JSON.stringify({ videoId: 'BackupSong01' }) })).json(); + expect(restored.ok).toBe(true); + expect((await db.getNotes('BackupSong01')).lyrics.data.lines[0].text).toBe('Before'); + expect(existsSync(restored.safetySnapshot)).toBe(true); + const dump = await app.request('/api/admin/backups/export/sqlite-dump', { headers: { 'x-admin': 'yes' } }); + expect(dump.status).toBe(200); + expect(await dump.text()).toContain('CREATE TABLE'); + }); }); diff --git a/tests/admin-backups.spec.js b/tests/admin-backups.spec.js new file mode 100644 index 0000000..79aed34 --- /dev/null +++ b/tests/admin-backups.spec.js @@ -0,0 +1,25 @@ +const { test, expect } = require('@playwright/test'); + +test('backups lists integrity, previews diff, creates a snapshot and confirms restore', async ({ page }) => { + const calls = []; + await page.route('**/api/**', async route => { + const request = route.request(), url = new URL(request.url()); calls.push({ path: url.pathname, method: request.method() }); + let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { ok: true, enabled: true, admin: true }; + if (url.pathname === '/api/admin/backups' && request.method() === 'GET') body = { ok: true, backups: [{ date: '2026-10-14', bytes: 1024, notes: 12, valid: true }] }; + if (url.pathname === '/api/admin/backups/2026-10-14') body = { ok: true, diff: { changed: 2, added: 1, removed: 0, preview: [{ key: 'song:lyrics', change: 'changed' }] } }; + if (url.pathname === '/api/admin/backups' && request.method() === 'POST') body = { ok: true, date: '2026-10-14', notes: 12 }; + if (url.pathname.endsWith('/restore')) body = { ok: true, restored: 12, safetySnapshot: '/data/backups/snapshots/safety.json' }; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + page.on('dialog', dialog => dialog.accept()); + await page.goto('/admin.html#backups'); + await expect(page.locator('#backupTable')).toContainText('Verified'); + await expect(page.locator('#backupDiff')).toContainText('2 changed'); + await expect(page.locator('a[href="/api/admin/backups/export/sqlite-dump"]')).toBeVisible(); + await page.locator('#backupNow').click(); + await expect(page.locator('#backupStatus')).toContainText('Saved 12 notes'); + await page.locator('#backupRestore').click(); + await expect.poll(() => calls.some(call => call.path.endsWith('/restore') && call.method === 'POST')).toBe(true); + await expect(page.locator('#backupStatus')).toContainText('Safety snapshot'); +}); From 11b10e38e20a71df6a6581897323bf855a332f03 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 02:10:02 +0800 Subject: [PATCH 7/9] Add searchable lyrics index --- frontend/admin.html | 40 +++++++++++++++- playwright.admin-audit.config.js | 2 +- server/admin-tools.js | 46 +++++++++++++++++++ server/admin-tools.test.js | 19 ++++++++ server/db.js | 76 +++++++++++++++++++++++++++++++ tests/admin-lyrics-search.spec.js | 28 ++++++++++++ 6 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 tests/admin-lyrics-search.spec.js diff --git a/frontend/admin.html b/frontend/admin.html index 11a1634..49a0751 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -558,6 +558,14 @@
+ +

Cache health

Review server copies by state and recency. Pinned copies stay out of automatic eviction.

@@ -778,6 +786,7 @@ editor: '', reports: '', quality: '', + 'lyric-search': '', cache: '', index: '', library: '', @@ -789,7 +798,7 @@ audit: '', access: '', }; - const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['lyric-search', 'Lyrics search'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -805,6 +814,7 @@ syncBars(); if (name === 'reports') loadReports(); if (name === 'quality') loadLyricsQuality(); + if (name === 'lyric-search') loadLyricsSearch(); if (name === 'cache') loadCacheHealth(); if (name === 'index') loadIndexHealth(); if (name === 'history') loadRevs(); @@ -1066,6 +1076,34 @@ } catch (error) { $('qualityStatus').className = 'msg err'; $('qualityStatus').textContent = error.message; } }); + /* ---------------- full-text lyrics search ---------------- */ + let lyricsSearchOffset = 0; + function lyricsSearchQuery() { + const params = new URLSearchParams({ q: $('lyricsSearchQuery').value.trim(), hasLyrics: $('lyricsSearchHasLyrics').value, hasChapters: $('lyricsSearchHasChapters').value, limit: '30', offset: String(lyricsSearchOffset) }); + return params.toString(); + } + const safeSnippet = value => esc(value || '').replaceAll('\u0001', '').replaceAll('\u0002', ''); + async function loadLyricsSearch() { + const table = $('lyricsSearchTable'); table.innerHTML = 'Searching saved lyrics…'; + try { + const result = await api('/api/admin/lyrics/search?' + lyricsSearchQuery()); if (!result.ok) throw Error(result.error || 'Lyrics search failed.'); + table.innerHTML = result.results.length ? `SongMatchChapters${result.results.map(row => `${esc(row.title || row.videoId)}
${esc(row.channel || row.videoId)}${safeSnippet(row.snippet)}${row.hasChapters ? 'Yes' : 'No'}`).join('')}` : 'No matching notes found.'; + $('lyricsSearchCount').textContent = `${result.total.toLocaleString()} matches · ${result.results.length ? lyricsSearchOffset + 1 : 0}–${lyricsSearchOffset + result.results.length}`; + $('lyricsSearchPrev').disabled = lyricsSearchOffset === 0; $('lyricsSearchNext').disabled = lyricsSearchOffset + result.results.length >= result.total; + $('lyricsSearchStatus').textContent = result.query ? `Searching “${result.query}” · ${result.hasLyrics} lyrics · ${result.hasChapters} chapters` : 'Showing saved lyrics. Enter words to search inside them.'; + } catch (error) { table.innerHTML = `${esc(error.message)}`; } + } + $('lyricsSearchForm').addEventListener('submit', event => { event.preventDefault(); lyricsSearchOffset = 0; loadLyricsSearch(); }); + $('lyricsSearchPrev').addEventListener('click', () => { lyricsSearchOffset = Math.max(0, lyricsSearchOffset - 30); loadLyricsSearch(); }); + $('lyricsSearchNext').addEventListener('click', () => { lyricsSearchOffset += 30; loadLyricsSearch(); }); + $('lyricsSearchTable').addEventListener('click', event => { const button = event.target.closest('[data-lyrics-search-open]'); if (button) leOpen(button.dataset.lyricsSearchOpen); }); + $('lyricsSearchRebuild').addEventListener('click', async () => { + $('lyricsSearchStatus').textContent = 'Rebuilding the lyric text index…'; + const result = await api('/api/admin/lyrics/search/rebuild', { method: 'POST', body: {} }); + $('lyricsSearchStatus').textContent = result.ok ? `Indexed ${result.indexed} saved lyric documents.` : result.error || 'Index rebuild failed.'; + if (result.ok) loadLyricsSearch(); + }); + /* ---------------- cache health and pinning ---------------- */ let cacheOffset = 0; async function loadCacheHealth() { diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js index bcdccd7..4fe7bf0 100644 --- a/playwright.admin-audit.config.js +++ b/playwright.admin-audit.config.js @@ -1,3 +1,3 @@ const { defineConfig } = require('@playwright/test'); const classic = require('./playwright.classic.config'); -module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index|reports|backups)\.spec\.js/ }); +module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index|reports|backups|lyrics-search)\.spec\.js/ }); diff --git a/server/admin-tools.js b/server/admin-tools.js index d8116a8..859bab1 100644 --- a/server/admin-tools.js +++ b/server/admin-tools.js @@ -156,6 +156,52 @@ export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, bac return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' }); }); + app.get('/api/admin/lyrics/search', adminAuth, async c => { + const query = String(c.req.query('q') || '').trim().slice(0, 120); + const hasLyrics = ['yes', 'no', 'all'].includes(c.req.query('hasLyrics')) ? c.req.query('hasLyrics') : 'yes'; + const hasChapters = ['yes', 'no', 'all'].includes(c.req.query('hasChapters')) ? c.req.query('hasChapters') : 'all'; + const limit = Math.max(1, Math.min(50, Number(c.req.query('limit')) || 30)); + const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); + let hits = [], total = 0; + if (query && hasLyrics !== 'no') { + const chapterOptions = { hasChapters: hasChapters === 'yes', withoutChapters: hasChapters === 'no' }; + hits = await db.searchNoteLyrics(query, { limit, offset, ...chapterOptions }); + total = await db.countNoteLyrics(query, chapterOptions); + } + const terms = query.normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase().match(/[\p{L}\p{N}_-]+/gu) || []; + if (!query || hasLyrics === 'no') { + const parts = []; + const args = []; + if (hasLyrics === 'yes') parts.push("l.video_id IS NOT NULL AND l.data<>'{}'"); + if (hasLyrics === 'no') parts.push('l.video_id IS NULL'); + if (hasChapters === 'yes') parts.push("EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters')"); + if (hasChapters === 'no') parts.push("NOT EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters')"); + for (const term of terms) { parts.push('lower(m.hay) LIKE ?'); args.push(`%${term}%`); } + const where = parts.length ? `WHERE ${parts.join(' AND ')}` : ''; + const count = await sql.execute({ sql: `SELECT COUNT(*) AS n FROM video_meta m LEFT JOIN video_notes l ON l.video_id=m.id AND l.kind='lyrics' ${where}`, args }); + const rows = await sql.execute({ sql: `SELECT m.id,m.card,l.data AS lyrics,l.rev,EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters') AS chapters FROM video_meta m LEFT JOIN video_notes l ON l.video_id=m.id AND l.kind='lyrics' ${where} ORDER BY m.updated_at DESC LIMIT ? OFFSET ?`, args: [...args, limit, offset] }); + total = number(count.rows[0]?.n); + hits = rows.rows.map(row => ({ videoId: row.id, snippet: parse(row.lyrics).lines?.slice(0, 3).map(line => line.text).join(' … ') || parse(row.card).title || row.id, rev: number(row.rev), hasChapters: !!row.chapters })); + } + if (hits.length && query && hasLyrics !== 'no') { + const ids = hits.map(item => item.videoId); + const rows = await sql.execute({ sql: `SELECT n.video_id,n.data,n.rev,m.card,EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=n.video_id AND ch.kind='chapters') AS chapters FROM video_notes n LEFT JOIN video_meta m ON m.id=n.video_id WHERE n.kind='lyrics' AND n.video_id IN (${ids.map(() => '?').join(',')})`, args: ids }); + const byId = new Map(rows.rows.map(row => [row.video_id, row])); + hits = hits.map(hit => { + const row = byId.get(hit.videoId), card = parse(row?.card); + return { ...hit, title: card.title || hit.videoId, channel: card.channel || '', rev: number(row?.rev), hasChapters: !!row?.chapters }; + }); + } + return c.json({ ok: true, query, hasLyrics, hasChapters, total, limit, offset, results: hits }, 200, { 'Cache-Control': 'no-store' }); + }); + + app.post('/api/admin/lyrics/search/rebuild', adminAuth, async c => { + if (typeof db.rebuildLyricsSearch !== 'function') return c.json({ ok: false, error: 'Lyrics search rebuild is unavailable.' }, 503); + const indexed = await db.rebuildLyricsSearch(); + await audit(db, 'admin', 'lyrics.search.rebuild', '', { indexed }); + return c.json({ ok: true, indexed }); + }); + app.get('/api/admin/cache/health', adminAuth, async c => { const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50)); const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); diff --git a/server/admin-tools.test.js b/server/admin-tools.test.js index 6643d32..669a6b8 100644 --- a/server/admin-tools.test.js +++ b/server/admin-tools.test.js @@ -93,4 +93,23 @@ describe('lyrics quality queue', () => { expect(dump.status).toBe(200); expect(await dump.text()).toContain('CREATE TABLE'); }); + + test('lyrics search is indexed on save, filters chapter state, and rebuilds under admin auth', async () => { + const id = 'SearchSong01'; + await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: [id, JSON.stringify({ title: 'Amazing grace sample', channel: 'Choir' }), 'amazing grace sample choir', 1, Date.now()] }); + await db.saveNote({ videoId: id, kind: 'lyrics', data: { lines: [{ t: 1, text: 'Amazing grace will lead me home' }] }, source: 'user', force: true }); + expect((await app.request('/api/admin/lyrics/search?q=amazing')).status).toBe(401); + const headers = { 'x-admin': 'yes' }; + let result = await (await app.request('/api/admin/lyrics/search?q=amazing+grace&hasChapters=no', { headers })).json(); + expect(result.results[0]).toMatchObject({ videoId: id, title: 'Amazing grace sample', hasChapters: false }); + expect(result.results[0].snippet).toContain('\u0001'); + await db.saveNote({ videoId: id, kind: 'chapters', data: { items: [{ t: 1, title: 'Verse' }] }, source: 'user', force: true }); + result = await (await app.request('/api/admin/lyrics/search?q=amazing&hasChapters=yes', { headers })).json(); + expect(result.results[0].videoId).toBe(id); + await db.saveNote({ videoId: id, kind: 'lyrics', data: { lines: [{ t: 1, text: 'Bright morning star' }] }, source: 'user', force: true }); + expect((await (await app.request('/api/admin/lyrics/search?q=amazing', { headers })).json()).total).toBe(0); + const rebuilt = await (await app.request('/api/admin/lyrics/search/rebuild', { method: 'POST', headers: { ...headers, 'content-type': 'application/json' }, body: '{}' })).json(); + expect(rebuilt.ok).toBe(true); + expect(rebuilt.indexed).toBeGreaterThanOrEqual(1); + }); }); diff --git a/server/db.js b/server/db.js index ddc6782..c89fc5a 100644 --- a/server/db.js +++ b/server/db.js @@ -20,6 +20,7 @@ const DB_PATH = process.env.DB_PATH || join(process.cwd(), 'data', 'ytplayer.db' export const db = createClient({ url: 'file:' + DB_PATH, }); +let lyricsFtsAvailable = false; // ---- Schema ---------------------------------------------------------------- @@ -279,6 +280,7 @@ export async function initDb() { CREATE INDEX IF NOT EXISTS idx_audit_actor_action ON audit_log (actor, action, ts DESC); `); await migrateVideoMeta(); + await initLyricsSearch(); try { await db.execute('ALTER TABLE media_cache ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0'); } catch { /* already migrated */ } // Uploads shared from devices (PWA share target): who sent it, and whether it // is listed in everyone's search (admin uploads) or reachable only by link. @@ -307,6 +309,75 @@ export async function initDb() { } } +function lyricsSearchContent(value) { + try { + const data = typeof value === 'string' ? JSON.parse(value) : value; + return [...(data?.lines || []).flatMap(line => [line?.text, line?.secondary, line?.phonetic]), ...(data?.tags || [])] + .filter(Boolean).join(' ').slice(0, 500_000); + } catch { return ''; } +} + +async function initLyricsSearch() { + await db.execute('CREATE TABLE IF NOT EXISTS lyrics_search_state (id INTEGER PRIMARY KEY CHECK(id=1), indexed_at INTEGER NOT NULL)'); + try { + await db.execute("CREATE VIRTUAL TABLE IF NOT EXISTS lyrics_search_fts USING fts5(video_id UNINDEXED, content, tokenize='unicode61 remove_diacritics 2')"); + lyricsFtsAvailable = true; + } catch { + await db.execute('CREATE TABLE IF NOT EXISTS lyrics_search_fts (video_id TEXT PRIMARY KEY, content TEXT NOT NULL)'); + await db.execute('CREATE INDEX IF NOT EXISTS idx_lyrics_search_id ON lyrics_search_fts(video_id)'); + lyricsFtsAvailable = false; + } + const state = await db.execute('SELECT id FROM lyrics_search_state WHERE id=1'); + if (state.rows.length) return; + await db.execute('DELETE FROM lyrics_search_fts'); + const rows = await db.execute("SELECT video_id,data FROM video_notes WHERE kind='lyrics'"); + for (let i = 0; i < rows.rows.length; i += 100) { + const batch = rows.rows.slice(i, i + 100).map(row => ({ + sql: lyricsFtsAvailable ? 'INSERT INTO lyrics_search_fts(video_id,content) VALUES(?,?)' : 'INSERT OR REPLACE INTO lyrics_search_fts(video_id,content) VALUES(?,?)', + args: [row.video_id, lyricsSearchContent(row.data)], + })); + if (batch.length) await db.batch(batch, 'write'); + } + await db.execute('INSERT INTO lyrics_search_state(id,indexed_at) VALUES(1,unixepoch())'); +} + +export async function searchNoteLyrics(query, { limit = 30, offset = 0, hasChapters = false, withoutChapters = false } = {}) { + const words = String(query || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').match(/[\p{L}\p{N}_-]+/gu) || []; + if (!words.length) return []; + const boundedLimit = Math.max(1, Math.min(100, Number(limit) || 30)); + const boundedOffset = Math.max(0, Math.min(1_000_000, Number(offset) || 0)); + const chapterFilter = hasChapters ? "AND EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : withoutChapters ? "AND NOT EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : ''; + if (lyricsFtsAvailable) { + const match = words.map(word => `"${word.replaceAll('"', '""')}"*`).join(' AND '); + const result = await db.execute({ sql: `SELECT video_id, snippet(lyrics_search_fts,1,char(1),char(2),' … ',18) AS snippet + FROM lyrics_search_fts WHERE lyrics_search_fts MATCH ? ${chapterFilter} ORDER BY bm25(lyrics_search_fts) LIMIT ? OFFSET ?`, args: [match, boundedLimit, boundedOffset] }); + return result.rows.map(row => ({ videoId: row.video_id, snippet: row.snippet })); + } + const filters = words.map(() => 'content LIKE ?').join(' AND '); + const result = await db.execute({ sql: `SELECT video_id,substr(content,1,220) AS snippet FROM lyrics_search_fts WHERE ${filters} ORDER BY video_id LIMIT ? OFFSET ?`, args: [...words.map(word => `%${word}%`), boundedLimit, boundedOffset] }); + return result.rows.map(row => ({ videoId: row.video_id, snippet: row.snippet })); +} + +export async function countNoteLyrics(query, { hasChapters = false, withoutChapters = false } = {}) { + const words = String(query || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').match(/[\p{L}\p{N}_-]+/gu) || []; + if (!words.length) return 0; + const chapterFilter = hasChapters ? "AND EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : withoutChapters ? "AND NOT EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : ''; + if (lyricsFtsAvailable) { + const match = words.map(word => `"${word.replaceAll('"', '""')}"*`).join(' AND '); + return Number((await db.execute({ sql: `SELECT COUNT(*) AS n FROM lyrics_search_fts WHERE lyrics_search_fts MATCH ? ${chapterFilter}`, args: [match] })).rows[0]?.n) || 0; + } + const filters = words.map(() => 'content LIKE ?').join(' AND '); + const result = await db.execute({ sql: `SELECT COUNT(*) AS n FROM lyrics_search_fts WHERE ${filters} ${chapterFilter}`, args: words.map(word => `%${word}%`) }); + return Number(result.rows[0]?.n) || 0; +} + +export async function rebuildLyricsSearch() { + await db.execute('DELETE FROM lyrics_search_fts'); + await db.execute('DELETE FROM lyrics_search_state'); + await initLyricsSearch(); + return Number((await db.execute("SELECT COUNT(*) AS n FROM video_notes WHERE kind='lyrics'")).rows[0]?.n) || 0; +} + const YOUTUBE_ID = /^[\w-]{11}$/; const folded = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase(); @@ -395,6 +466,11 @@ export async function saveNote({ videoId, kind, data, baseRev, source, updatedBy VALUES (?, ?, ?, ?, ?, ?, unixepoch())`, args: [videoId, kind, rev, json, source, updatedBy || null], }); + if (kind === 'lyrics') { + await tx.execute({ sql: 'DELETE FROM lyrics_search_fts WHERE video_id=?', args: [videoId] }); + const content = lyricsSearchContent(data); + if (content) await tx.execute({ sql: 'INSERT INTO lyrics_search_fts(video_id,content) VALUES(?,?)', args: [videoId, content] }); + } await tx.commit(); return { ok: true, rev }; } catch (err) { diff --git a/tests/admin-lyrics-search.spec.js b/tests/admin-lyrics-search.spec.js new file mode 100644 index 0000000..9784f4a --- /dev/null +++ b/tests/admin-lyrics-search.spec.js @@ -0,0 +1,28 @@ +const { test, expect } = require('@playwright/test'); + +test('lyrics full-text search highlights matches, filters, rebuilds and opens the editor', async ({ page }) => { + const requests = []; + await page.route('**/api/**', async route => { + const request = route.request(), url = new URL(request.url()); requests.push({ path: url.pathname, query: url.search, method: request.method() }); + let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { ok: true, enabled: true, admin: true }; + if (url.pathname === '/api/admin/lyrics/search' && request.method() === 'GET') body = { ok: true, query: url.searchParams.get('q') || '', hasLyrics: 'yes', hasChapters: 'all', total: 1, limit: 30, offset: 0, results: [{ videoId: 'dQw4w9WgXcQ', title: 'Amazing grace', channel: 'Choir', snippet: '\u0001Amazing grace\u0002 will lead me home', hasChapters: true, rev: 2 }] }; + if (url.pathname === '/api/admin/lyrics/search/rebuild') body = { ok: true, indexed: 42 }; + if (url.pathname === '/api/notes/dQw4w9WgXcQ') body = { ok: true, lyrics: { rev: 2, data: { lines: [{ t: 1, text: 'Amazing grace', kind: 'line' }] } }, chapters: null }; + if (url.pathname === '/api/streams') body = { ok: true, data: { meta: { title: 'Amazing grace', duration: 30 }, qualities: [] } }; + if (url.pathname === '/api/admin/flags') body = { ok: true, open: 0, flags: [] }; + if (url.pathname.startsWith('/api/admin/transcriptions/')) body = { ok: true, job: null }; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.goto('/admin.html#lyric-search'); + await expect(page.locator('#lyricsSearchTable')).toContainText('Amazing grace'); + await expect(page.locator('#lyricsSearchTable mark')).toHaveText('Amazing grace'); + await page.locator('#lyricsSearchQuery').fill('amazing grace'); await page.locator('#lyricsSearchHasChapters').selectOption('yes'); + await page.locator('#lyricsSearchForm button[type=submit]').click(); + await expect.poll(() => requests.some(item => item.path === '/api/admin/lyrics/search' && item.query.includes('hasChapters=yes'))).toBe(true); + await page.locator('#lyricsSearchRebuild').click(); + await expect.poll(() => requests.some(item => item.path === '/api/admin/lyrics/search/rebuild' && item.method === 'POST')).toBe(true); + await expect(page.locator('#lyricsSearchTable')).toContainText('Amazing grace'); + await page.locator('[data-lyrics-search-open="dQw4w9WgXcQ"]').click(); + await expect(page.locator('#songTitle')).toHaveText('Amazing grace'); +}); From 555027a61c8e1966a792f76146cb1a49ac0570b2 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 02:15:00 +0800 Subject: [PATCH 8/9] Count all pending cache states --- frontend/admin.html | 2 +- tests/admin-cache.spec.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/admin.html b/frontend/admin.html index 49a0751..bb10741 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -1111,7 +1111,7 @@ try { const j = await api(`/api/admin/cache/health?limit=50&offset=${cacheOffset}`); if (!j.ok) throw Error(j.error || 'Cache health unavailable.'); - const ready = j.statuses.find(x => x.status === 'ready') || {}, pending = j.statuses.find(x => ['queued', 'downloading', 'validating'].includes(x.status)), failed = j.statuses.find(x => x.status === 'failed') || {}; + const ready = j.statuses.find(x => x.status === 'ready') || {}, pending = j.statuses.filter(x => ['queued', 'downloading', 'validating'].includes(x.status)).reduce((sum, item) => ({ count: sum.count + Number(item.count || 0), bytes: sum.bytes + Number(item.bytes || 0) }), { count: 0, bytes: 0 }), failed = j.statuses.find(x => x.status === 'failed') || {}; $('cacheMetrics').innerHTML = [ ['Cache used', `${bytes(j.bytes)} / ${bytes(j.budget)}`, `${Math.round(j.utilization * 100)}% of configured budget`], ['Ready', Number(ready.count || 0).toLocaleString(), bytes(ready.bytes || 0)], diff --git a/tests/admin-cache.spec.js b/tests/admin-cache.spec.js index bfe7f84..120da4b 100644 --- a/tests/admin-cache.spec.js +++ b/tests/admin-cache.spec.js @@ -5,7 +5,7 @@ test('cache actions pin, unpin, confirm eviction and request a source re-downloa await page.route('**/api/**', async route => { const req = route.request(), url = new URL(req.url()); let body = { ok: true }; if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true }; - else if (url.pathname === '/api/admin/cache/health') body = { ok: true, total: 1, offset: 0, budget: 1000, pinnedBudget: 900, bytes: 700, pinnedBytes: state.pinned ? 700 : 0, utilization: .7, statuses: [{ status: 'ready', count: 1, bytes: 700 }], media: [{ id: 'CacheItem001', status: 'ready', bytes: 700, lastAccess: 1791900000000, gen: 4, pinned: state.pinned }] }; + else if (url.pathname === '/api/admin/cache/health') body = { ok: true, total: 1, offset: 0, budget: 1000, pinnedBudget: 900, bytes: 950, pinnedBytes: state.pinned ? 700 : 0, utilization: .95, statuses: [{ status: 'ready', count: 1, bytes: 700 }, { status: 'queued', count: 2, bytes: 100 }, { status: 'downloading', count: 3, bytes: 150 }, { status: 'failed', count: 1, bytes: 0 }], media: [{ id: 'CacheItem001', status: 'ready', bytes: 700, lastAccess: 1791900000000, gen: 4, pinned: state.pinned }] }; else if (url.pathname === '/api/admin/cache/CacheItem001/pin' && req.method() === 'POST') { state.pinned = req.postDataJSON().pinned; body.pinned = state.pinned; } else if (url.pathname === '/api/admin/cache/CacheItem001/evict' && req.method() === 'POST') state.evicted = true; else if (url.pathname === '/api/admin/media/CacheItem001/redownload' && req.method() === 'POST') state.redownloaded = true; @@ -16,6 +16,7 @@ test('cache actions pin, unpin, confirm eviction and request a source re-downloa }); await page.goto('/admin.html#cache'); await expect(page.locator('#cacheTable')).toContainText('CacheItem001'); + await expect(page.locator('#cacheMetrics')).toContainText('5 / 1'); await page.locator('[data-cache-pin]').click(); await expect(page.locator('[data-cache-pin]')).toHaveText('Unpin'); await page.locator('[data-cache-pin]').click(); await expect(page.locator('[data-cache-pin]')).toHaveText('Pin'); page.once('dialog', dialog => dialog.accept()); await page.locator('[data-cache-evict]').click(); From 703aef617bbd90c59ea4deb15e63ee9e1fb33931 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 02:36:30 +0800 Subject: [PATCH 9/9] Add resumable bulk lyrics operations --- frontend/admin.html | 184 ++++++++++++++---- plans/admin-tools-shots/audit.png | Bin 0 -> 81969 bytes plans/admin-tools-shots/backups.png | Bin 0 -> 92367 bytes .../admin-tools-shots/bulk-contrast-1280.png | Bin 0 -> 53095 bytes plans/admin-tools-shots/bulk-contrast-390.png | Bin 0 -> 53823 bytes plans/admin-tools-shots/bulk-dark-1280.png | Bin 0 -> 67967 bytes plans/admin-tools-shots/bulk-dark-390.png | Bin 0 -> 65293 bytes plans/admin-tools-shots/bulk-light-1280.png | Bin 0 -> 55920 bytes plans/admin-tools-shots/bulk-light-390.png | Bin 0 -> 57589 bytes plans/admin-tools-shots/cache.png | Bin 0 -> 86454 bytes plans/admin-tools-shots/index.png | Bin 0 -> 126806 bytes plans/admin-tools-shots/library.png | Bin 0 -> 166551 bytes plans/admin-tools-shots/lyric-search.png | Bin 0 -> 74732 bytes plans/admin-tools-shots/quality.png | Bin 0 -> 72917 bytes plans/admin-tools-shots/reports.png | Bin 0 -> 78630 bytes playwright.admin-audit.config.js | 2 +- server/admin-tools.js | 172 +++++++++++++++- server/admin-tools.test.js | 48 ++++- server/db.js | 7 + server/notes.js | 30 ++- server/notes.test.js | 19 ++ server/server.js | 3 +- server/transcriptions.js | 23 ++- tests/admin-bulk-lyrics.spec.js | 43 ++++ tests/admin-tools-shots.spec.js | 71 +++++++ 25 files changed, 547 insertions(+), 55 deletions(-) create mode 100644 plans/admin-tools-shots/audit.png create mode 100644 plans/admin-tools-shots/backups.png create mode 100644 plans/admin-tools-shots/bulk-contrast-1280.png create mode 100644 plans/admin-tools-shots/bulk-contrast-390.png create mode 100644 plans/admin-tools-shots/bulk-dark-1280.png create mode 100644 plans/admin-tools-shots/bulk-dark-390.png create mode 100644 plans/admin-tools-shots/bulk-light-1280.png create mode 100644 plans/admin-tools-shots/bulk-light-390.png create mode 100644 plans/admin-tools-shots/cache.png create mode 100644 plans/admin-tools-shots/index.png create mode 100644 plans/admin-tools-shots/library.png create mode 100644 plans/admin-tools-shots/lyric-search.png create mode 100644 plans/admin-tools-shots/quality.png create mode 100644 plans/admin-tools-shots/reports.png create mode 100644 tests/admin-bulk-lyrics.spec.js create mode 100644 tests/admin-tools-shots.spec.js diff --git a/frontend/admin.html b/frontend/admin.html index bb10741..e1dc8f3 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -16,29 +16,61 @@