diff --git a/frontend/admin.html b/frontend/admin.html index 11a1634..49a0751 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -558,6 +558,14 @@ + +

Cache health

Review server copies by state and recency. Pinned copies stay out of automatic eviction.

@@ -778,6 +786,7 @@ editor: '', reports: '', quality: '', + 'lyric-search': '', cache: '', index: '', library: '', @@ -789,7 +798,7 @@ audit: '', access: '', }; - const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['lyric-search', 'Lyrics search'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -805,6 +814,7 @@ syncBars(); if (name === 'reports') loadReports(); if (name === 'quality') loadLyricsQuality(); + if (name === 'lyric-search') loadLyricsSearch(); if (name === 'cache') loadCacheHealth(); if (name === 'index') loadIndexHealth(); if (name === 'history') loadRevs(); @@ -1066,6 +1076,34 @@ } catch (error) { $('qualityStatus').className = 'msg err'; $('qualityStatus').textContent = error.message; } }); + /* ---------------- full-text lyrics search ---------------- */ + let lyricsSearchOffset = 0; + function lyricsSearchQuery() { + const params = new URLSearchParams({ q: $('lyricsSearchQuery').value.trim(), hasLyrics: $('lyricsSearchHasLyrics').value, hasChapters: $('lyricsSearchHasChapters').value, limit: '30', offset: String(lyricsSearchOffset) }); + return params.toString(); + } + const safeSnippet = value => esc(value || '').replaceAll('\u0001', '').replaceAll('\u0002', ''); + async function loadLyricsSearch() { + const table = $('lyricsSearchTable'); table.innerHTML = 'Searching saved lyrics…'; + try { + const result = await api('/api/admin/lyrics/search?' + lyricsSearchQuery()); if (!result.ok) throw Error(result.error || 'Lyrics search failed.'); + table.innerHTML = result.results.length ? `SongMatchChapters${result.results.map(row => `${esc(row.title || row.videoId)}
${esc(row.channel || row.videoId)}${safeSnippet(row.snippet)}${row.hasChapters ? 'Yes' : 'No'}`).join('')}` : 'No matching notes found.'; + $('lyricsSearchCount').textContent = `${result.total.toLocaleString()} matches · ${result.results.length ? lyricsSearchOffset + 1 : 0}–${lyricsSearchOffset + result.results.length}`; + $('lyricsSearchPrev').disabled = lyricsSearchOffset === 0; $('lyricsSearchNext').disabled = lyricsSearchOffset + result.results.length >= result.total; + $('lyricsSearchStatus').textContent = result.query ? `Searching “${result.query}” · ${result.hasLyrics} lyrics · ${result.hasChapters} chapters` : 'Showing saved lyrics. Enter words to search inside them.'; + } catch (error) { table.innerHTML = `${esc(error.message)}`; } + } + $('lyricsSearchForm').addEventListener('submit', event => { event.preventDefault(); lyricsSearchOffset = 0; loadLyricsSearch(); }); + $('lyricsSearchPrev').addEventListener('click', () => { lyricsSearchOffset = Math.max(0, lyricsSearchOffset - 30); loadLyricsSearch(); }); + $('lyricsSearchNext').addEventListener('click', () => { lyricsSearchOffset += 30; loadLyricsSearch(); }); + $('lyricsSearchTable').addEventListener('click', event => { const button = event.target.closest('[data-lyrics-search-open]'); if (button) leOpen(button.dataset.lyricsSearchOpen); }); + $('lyricsSearchRebuild').addEventListener('click', async () => { + $('lyricsSearchStatus').textContent = 'Rebuilding the lyric text index…'; + const result = await api('/api/admin/lyrics/search/rebuild', { method: 'POST', body: {} }); + $('lyricsSearchStatus').textContent = result.ok ? `Indexed ${result.indexed} saved lyric documents.` : result.error || 'Index rebuild failed.'; + if (result.ok) loadLyricsSearch(); + }); + /* ---------------- cache health and pinning ---------------- */ let cacheOffset = 0; async function loadCacheHealth() { diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js index bcdccd7..4fe7bf0 100644 --- a/playwright.admin-audit.config.js +++ b/playwright.admin-audit.config.js @@ -1,3 +1,3 @@ const { defineConfig } = require('@playwright/test'); const classic = require('./playwright.classic.config'); -module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index|reports|backups)\.spec\.js/ }); +module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index|reports|backups|lyrics-search)\.spec\.js/ }); diff --git a/server/admin-tools.js b/server/admin-tools.js index d8116a8..859bab1 100644 --- a/server/admin-tools.js +++ b/server/admin-tools.js @@ -156,6 +156,52 @@ export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, bac return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' }); }); + app.get('/api/admin/lyrics/search', adminAuth, async c => { + const query = String(c.req.query('q') || '').trim().slice(0, 120); + const hasLyrics = ['yes', 'no', 'all'].includes(c.req.query('hasLyrics')) ? c.req.query('hasLyrics') : 'yes'; + const hasChapters = ['yes', 'no', 'all'].includes(c.req.query('hasChapters')) ? c.req.query('hasChapters') : 'all'; + const limit = Math.max(1, Math.min(50, Number(c.req.query('limit')) || 30)); + const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); + let hits = [], total = 0; + if (query && hasLyrics !== 'no') { + const chapterOptions = { hasChapters: hasChapters === 'yes', withoutChapters: hasChapters === 'no' }; + hits = await db.searchNoteLyrics(query, { limit, offset, ...chapterOptions }); + total = await db.countNoteLyrics(query, chapterOptions); + } + const terms = query.normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase().match(/[\p{L}\p{N}_-]+/gu) || []; + if (!query || hasLyrics === 'no') { + const parts = []; + const args = []; + if (hasLyrics === 'yes') parts.push("l.video_id IS NOT NULL AND l.data<>'{}'"); + if (hasLyrics === 'no') parts.push('l.video_id IS NULL'); + if (hasChapters === 'yes') parts.push("EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters')"); + if (hasChapters === 'no') parts.push("NOT EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters')"); + for (const term of terms) { parts.push('lower(m.hay) LIKE ?'); args.push(`%${term}%`); } + const where = parts.length ? `WHERE ${parts.join(' AND ')}` : ''; + const count = await sql.execute({ sql: `SELECT COUNT(*) AS n FROM video_meta m LEFT JOIN video_notes l ON l.video_id=m.id AND l.kind='lyrics' ${where}`, args }); + const rows = await sql.execute({ sql: `SELECT m.id,m.card,l.data AS lyrics,l.rev,EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters') AS chapters FROM video_meta m LEFT JOIN video_notes l ON l.video_id=m.id AND l.kind='lyrics' ${where} ORDER BY m.updated_at DESC LIMIT ? OFFSET ?`, args: [...args, limit, offset] }); + total = number(count.rows[0]?.n); + hits = rows.rows.map(row => ({ videoId: row.id, snippet: parse(row.lyrics).lines?.slice(0, 3).map(line => line.text).join(' … ') || parse(row.card).title || row.id, rev: number(row.rev), hasChapters: !!row.chapters })); + } + if (hits.length && query && hasLyrics !== 'no') { + const ids = hits.map(item => item.videoId); + const rows = await sql.execute({ sql: `SELECT n.video_id,n.data,n.rev,m.card,EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=n.video_id AND ch.kind='chapters') AS chapters FROM video_notes n LEFT JOIN video_meta m ON m.id=n.video_id WHERE n.kind='lyrics' AND n.video_id IN (${ids.map(() => '?').join(',')})`, args: ids }); + const byId = new Map(rows.rows.map(row => [row.video_id, row])); + hits = hits.map(hit => { + const row = byId.get(hit.videoId), card = parse(row?.card); + return { ...hit, title: card.title || hit.videoId, channel: card.channel || '', rev: number(row?.rev), hasChapters: !!row?.chapters }; + }); + } + return c.json({ ok: true, query, hasLyrics, hasChapters, total, limit, offset, results: hits }, 200, { 'Cache-Control': 'no-store' }); + }); + + app.post('/api/admin/lyrics/search/rebuild', adminAuth, async c => { + if (typeof db.rebuildLyricsSearch !== 'function') return c.json({ ok: false, error: 'Lyrics search rebuild is unavailable.' }, 503); + const indexed = await db.rebuildLyricsSearch(); + await audit(db, 'admin', 'lyrics.search.rebuild', '', { indexed }); + return c.json({ ok: true, indexed }); + }); + app.get('/api/admin/cache/health', adminAuth, async c => { const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50)); const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); diff --git a/server/admin-tools.test.js b/server/admin-tools.test.js index 6643d32..669a6b8 100644 --- a/server/admin-tools.test.js +++ b/server/admin-tools.test.js @@ -93,4 +93,23 @@ describe('lyrics quality queue', () => { expect(dump.status).toBe(200); expect(await dump.text()).toContain('CREATE TABLE'); }); + + test('lyrics search is indexed on save, filters chapter state, and rebuilds under admin auth', async () => { + const id = 'SearchSong01'; + await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: [id, JSON.stringify({ title: 'Amazing grace sample', channel: 'Choir' }), 'amazing grace sample choir', 1, Date.now()] }); + await db.saveNote({ videoId: id, kind: 'lyrics', data: { lines: [{ t: 1, text: 'Amazing grace will lead me home' }] }, source: 'user', force: true }); + expect((await app.request('/api/admin/lyrics/search?q=amazing')).status).toBe(401); + const headers = { 'x-admin': 'yes' }; + let result = await (await app.request('/api/admin/lyrics/search?q=amazing+grace&hasChapters=no', { headers })).json(); + expect(result.results[0]).toMatchObject({ videoId: id, title: 'Amazing grace sample', hasChapters: false }); + expect(result.results[0].snippet).toContain('\u0001'); + await db.saveNote({ videoId: id, kind: 'chapters', data: { items: [{ t: 1, title: 'Verse' }] }, source: 'user', force: true }); + result = await (await app.request('/api/admin/lyrics/search?q=amazing&hasChapters=yes', { headers })).json(); + expect(result.results[0].videoId).toBe(id); + await db.saveNote({ videoId: id, kind: 'lyrics', data: { lines: [{ t: 1, text: 'Bright morning star' }] }, source: 'user', force: true }); + expect((await (await app.request('/api/admin/lyrics/search?q=amazing', { headers })).json()).total).toBe(0); + const rebuilt = await (await app.request('/api/admin/lyrics/search/rebuild', { method: 'POST', headers: { ...headers, 'content-type': 'application/json' }, body: '{}' })).json(); + expect(rebuilt.ok).toBe(true); + expect(rebuilt.indexed).toBeGreaterThanOrEqual(1); + }); }); diff --git a/server/db.js b/server/db.js index ddc6782..c89fc5a 100644 --- a/server/db.js +++ b/server/db.js @@ -20,6 +20,7 @@ const DB_PATH = process.env.DB_PATH || join(process.cwd(), 'data', 'ytplayer.db' export const db = createClient({ url: 'file:' + DB_PATH, }); +let lyricsFtsAvailable = false; // ---- Schema ---------------------------------------------------------------- @@ -279,6 +280,7 @@ export async function initDb() { CREATE INDEX IF NOT EXISTS idx_audit_actor_action ON audit_log (actor, action, ts DESC); `); await migrateVideoMeta(); + await initLyricsSearch(); try { await db.execute('ALTER TABLE media_cache ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0'); } catch { /* already migrated */ } // Uploads shared from devices (PWA share target): who sent it, and whether it // is listed in everyone's search (admin uploads) or reachable only by link. @@ -307,6 +309,75 @@ export async function initDb() { } } +function lyricsSearchContent(value) { + try { + const data = typeof value === 'string' ? JSON.parse(value) : value; + return [...(data?.lines || []).flatMap(line => [line?.text, line?.secondary, line?.phonetic]), ...(data?.tags || [])] + .filter(Boolean).join(' ').slice(0, 500_000); + } catch { return ''; } +} + +async function initLyricsSearch() { + await db.execute('CREATE TABLE IF NOT EXISTS lyrics_search_state (id INTEGER PRIMARY KEY CHECK(id=1), indexed_at INTEGER NOT NULL)'); + try { + await db.execute("CREATE VIRTUAL TABLE IF NOT EXISTS lyrics_search_fts USING fts5(video_id UNINDEXED, content, tokenize='unicode61 remove_diacritics 2')"); + lyricsFtsAvailable = true; + } catch { + await db.execute('CREATE TABLE IF NOT EXISTS lyrics_search_fts (video_id TEXT PRIMARY KEY, content TEXT NOT NULL)'); + await db.execute('CREATE INDEX IF NOT EXISTS idx_lyrics_search_id ON lyrics_search_fts(video_id)'); + lyricsFtsAvailable = false; + } + const state = await db.execute('SELECT id FROM lyrics_search_state WHERE id=1'); + if (state.rows.length) return; + await db.execute('DELETE FROM lyrics_search_fts'); + const rows = await db.execute("SELECT video_id,data FROM video_notes WHERE kind='lyrics'"); + for (let i = 0; i < rows.rows.length; i += 100) { + const batch = rows.rows.slice(i, i + 100).map(row => ({ + sql: lyricsFtsAvailable ? 'INSERT INTO lyrics_search_fts(video_id,content) VALUES(?,?)' : 'INSERT OR REPLACE INTO lyrics_search_fts(video_id,content) VALUES(?,?)', + args: [row.video_id, lyricsSearchContent(row.data)], + })); + if (batch.length) await db.batch(batch, 'write'); + } + await db.execute('INSERT INTO lyrics_search_state(id,indexed_at) VALUES(1,unixepoch())'); +} + +export async function searchNoteLyrics(query, { limit = 30, offset = 0, hasChapters = false, withoutChapters = false } = {}) { + const words = String(query || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').match(/[\p{L}\p{N}_-]+/gu) || []; + if (!words.length) return []; + const boundedLimit = Math.max(1, Math.min(100, Number(limit) || 30)); + const boundedOffset = Math.max(0, Math.min(1_000_000, Number(offset) || 0)); + const chapterFilter = hasChapters ? "AND EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : withoutChapters ? "AND NOT EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : ''; + if (lyricsFtsAvailable) { + const match = words.map(word => `"${word.replaceAll('"', '""')}"*`).join(' AND '); + const result = await db.execute({ sql: `SELECT video_id, snippet(lyrics_search_fts,1,char(1),char(2),' … ',18) AS snippet + FROM lyrics_search_fts WHERE lyrics_search_fts MATCH ? ${chapterFilter} ORDER BY bm25(lyrics_search_fts) LIMIT ? OFFSET ?`, args: [match, boundedLimit, boundedOffset] }); + return result.rows.map(row => ({ videoId: row.video_id, snippet: row.snippet })); + } + const filters = words.map(() => 'content LIKE ?').join(' AND '); + const result = await db.execute({ sql: `SELECT video_id,substr(content,1,220) AS snippet FROM lyrics_search_fts WHERE ${filters} ORDER BY video_id LIMIT ? OFFSET ?`, args: [...words.map(word => `%${word}%`), boundedLimit, boundedOffset] }); + return result.rows.map(row => ({ videoId: row.video_id, snippet: row.snippet })); +} + +export async function countNoteLyrics(query, { hasChapters = false, withoutChapters = false } = {}) { + const words = String(query || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').match(/[\p{L}\p{N}_-]+/gu) || []; + if (!words.length) return 0; + const chapterFilter = hasChapters ? "AND EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : withoutChapters ? "AND NOT EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : ''; + if (lyricsFtsAvailable) { + const match = words.map(word => `"${word.replaceAll('"', '""')}"*`).join(' AND '); + return Number((await db.execute({ sql: `SELECT COUNT(*) AS n FROM lyrics_search_fts WHERE lyrics_search_fts MATCH ? ${chapterFilter}`, args: [match] })).rows[0]?.n) || 0; + } + const filters = words.map(() => 'content LIKE ?').join(' AND '); + const result = await db.execute({ sql: `SELECT COUNT(*) AS n FROM lyrics_search_fts WHERE ${filters} ${chapterFilter}`, args: words.map(word => `%${word}%`) }); + return Number(result.rows[0]?.n) || 0; +} + +export async function rebuildLyricsSearch() { + await db.execute('DELETE FROM lyrics_search_fts'); + await db.execute('DELETE FROM lyrics_search_state'); + await initLyricsSearch(); + return Number((await db.execute("SELECT COUNT(*) AS n FROM video_notes WHERE kind='lyrics'")).rows[0]?.n) || 0; +} + const YOUTUBE_ID = /^[\w-]{11}$/; const folded = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase(); @@ -395,6 +466,11 @@ export async function saveNote({ videoId, kind, data, baseRev, source, updatedBy VALUES (?, ?, ?, ?, ?, ?, unixepoch())`, args: [videoId, kind, rev, json, source, updatedBy || null], }); + if (kind === 'lyrics') { + await tx.execute({ sql: 'DELETE FROM lyrics_search_fts WHERE video_id=?', args: [videoId] }); + const content = lyricsSearchContent(data); + if (content) await tx.execute({ sql: 'INSERT INTO lyrics_search_fts(video_id,content) VALUES(?,?)', args: [videoId, content] }); + } await tx.commit(); return { ok: true, rev }; } catch (err) { diff --git a/tests/admin-lyrics-search.spec.js b/tests/admin-lyrics-search.spec.js new file mode 100644 index 0000000..9784f4a --- /dev/null +++ b/tests/admin-lyrics-search.spec.js @@ -0,0 +1,28 @@ +const { test, expect } = require('@playwright/test'); + +test('lyrics full-text search highlights matches, filters, rebuilds and opens the editor', async ({ page }) => { + const requests = []; + await page.route('**/api/**', async route => { + const request = route.request(), url = new URL(request.url()); requests.push({ path: url.pathname, query: url.search, method: request.method() }); + let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { ok: true, enabled: true, admin: true }; + if (url.pathname === '/api/admin/lyrics/search' && request.method() === 'GET') body = { ok: true, query: url.searchParams.get('q') || '', hasLyrics: 'yes', hasChapters: 'all', total: 1, limit: 30, offset: 0, results: [{ videoId: 'dQw4w9WgXcQ', title: 'Amazing grace', channel: 'Choir', snippet: '\u0001Amazing grace\u0002 will lead me home', hasChapters: true, rev: 2 }] }; + if (url.pathname === '/api/admin/lyrics/search/rebuild') body = { ok: true, indexed: 42 }; + if (url.pathname === '/api/notes/dQw4w9WgXcQ') body = { ok: true, lyrics: { rev: 2, data: { lines: [{ t: 1, text: 'Amazing grace', kind: 'line' }] } }, chapters: null }; + if (url.pathname === '/api/streams') body = { ok: true, data: { meta: { title: 'Amazing grace', duration: 30 }, qualities: [] } }; + if (url.pathname === '/api/admin/flags') body = { ok: true, open: 0, flags: [] }; + if (url.pathname.startsWith('/api/admin/transcriptions/')) body = { ok: true, job: null }; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.goto('/admin.html#lyric-search'); + await expect(page.locator('#lyricsSearchTable')).toContainText('Amazing grace'); + await expect(page.locator('#lyricsSearchTable mark')).toHaveText('Amazing grace'); + await page.locator('#lyricsSearchQuery').fill('amazing grace'); await page.locator('#lyricsSearchHasChapters').selectOption('yes'); + await page.locator('#lyricsSearchForm button[type=submit]').click(); + await expect.poll(() => requests.some(item => item.path === '/api/admin/lyrics/search' && item.query.includes('hasChapters=yes'))).toBe(true); + await page.locator('#lyricsSearchRebuild').click(); + await expect.poll(() => requests.some(item => item.path === '/api/admin/lyrics/search/rebuild' && item.method === 'POST')).toBe(true); + await expect(page.locator('#lyricsSearchTable')).toContainText('Amazing grace'); + await page.locator('[data-lyrics-search-open="dQw4w9WgXcQ"]').click(); + await expect(page.locator('#songTitle')).toHaveText('Amazing grace'); +});