josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add searchable lyrics index

Browse Source
This commit is contained in:
Jonathan Sykes
2026-10-14 02:10:02 +08:00
parent e3821621f3
commit 11b10e38e2
6 changed files with 209 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
server/admin-tools.js
View File

@@ -156,6 +156,52 @@ export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, bac
return c.json({ ok: true, minLines, category, offset, limit, total: number(count.rows[0]?.total), songs }, 200, { 'Cache-Control': 'no-store' });
});
app.get('/api/admin/lyrics/search', adminAuth, async c => {
const query = String(c.req.query('q') || '').trim().slice(0, 120);
const hasLyrics = ['yes', 'no', 'all'].includes(c.req.query('hasLyrics')) ? c.req.query('hasLyrics') : 'yes';
const hasChapters = ['yes', 'no', 'all'].includes(c.req.query('hasChapters')) ? c.req.query('hasChapters') : 'all';
const limit = Math.max(1, Math.min(50, Number(c.req.query('limit')) || 30));
const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
let hits = [], total = 0;
if (query && hasLyrics !== 'no') {
const chapterOptions = { hasChapters: hasChapters === 'yes', withoutChapters: hasChapters === 'no' };
hits = await db.searchNoteLyrics(query, { limit, offset, ...chapterOptions });
total = await db.countNoteLyrics(query, chapterOptions);
}
const terms = query.normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase().match(/[\p{L}\p{N}_-]+/gu) || [];
if (!query || hasLyrics === 'no') {
const parts = [];
const args = [];
if (hasLyrics === 'yes') parts.push("l.video_id IS NOT NULL AND l.data<>'{}'");
if (hasLyrics === 'no') parts.push('l.video_id IS NULL');
if (hasChapters === 'yes') parts.push("EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters')");
if (hasChapters === 'no') parts.push("NOT EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters')");
for (const term of terms) { parts.push('lower(m.hay) LIKE ?'); args.push(`%${term}%`); }
const where = parts.length ? `WHERE ${parts.join(' AND ')}` : '';
const count = await sql.execute({ sql: `SELECT COUNT(*) AS n FROM video_meta m LEFT JOIN video_notes l ON l.video_id=m.id AND l.kind='lyrics' ${where}`, args });
const rows = await sql.execute({ sql: `SELECT m.id,m.card,l.data AS lyrics,l.rev,EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=m.id AND ch.kind='chapters') AS chapters FROM video_meta m LEFT JOIN video_notes l ON l.video_id=m.id AND l.kind='lyrics' ${where} ORDER BY m.updated_at DESC LIMIT ? OFFSET ?`, args: [...args, limit, offset] });
total = number(count.rows[0]?.n);
hits = rows.rows.map(row => ({ videoId: row.id, snippet: parse(row.lyrics).lines?.slice(0, 3).map(line => line.text).join(' … ') || parse(row.card).title || row.id, rev: number(row.rev), hasChapters: !!row.chapters }));
}
if (hits.length && query && hasLyrics !== 'no') {
const ids = hits.map(item => item.videoId);
const rows = await sql.execute({ sql: `SELECT n.video_id,n.data,n.rev,m.card,EXISTS(SELECT 1 FROM video_notes ch WHERE ch.video_id=n.video_id AND ch.kind='chapters') AS chapters FROM video_notes n LEFT JOIN video_meta m ON m.id=n.video_id WHERE n.kind='lyrics' AND n.video_id IN (${ids.map(() => '?').join(',')})`, args: ids });
const byId = new Map(rows.rows.map(row => [row.video_id, row]));
hits = hits.map(hit => {
const row = byId.get(hit.videoId), card = parse(row?.card);
return { ...hit, title: card.title || hit.videoId, channel: card.channel || '', rev: number(row?.rev), hasChapters: !!row?.chapters };
});
}
return c.json({ ok: true, query, hasLyrics, hasChapters, total, limit, offset, results: hits }, 200, { 'Cache-Control': 'no-store' });
});
app.post('/api/admin/lyrics/search/rebuild', adminAuth, async c => {
if (typeof db.rebuildLyricsSearch !== 'function') return c.json({ ok: false, error: 'Lyrics search rebuild is unavailable.' }, 503);
const indexed = await db.rebuildLyricsSearch();
await audit(db, 'admin', 'lyrics.search.rebuild', '', { indexed });
return c.json({ ok: true, indexed });
});
app.get('/api/admin/cache/health', adminAuth, async c => {
const limit = Math.max(1, Math.min(100, Number(c.req.query('limit')) || 50));
const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));

19
server/admin-tools.test.js
View File

@@ -93,4 +93,23 @@ describe('lyrics quality queue', () => {
expect(dump.status).toBe(200);
expect(await dump.text()).toContain('CREATE TABLE');
});
test('lyrics search is indexed on save, filters chapter state, and rebuilds under admin auth', async () => {
const id = 'SearchSong01';
await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: [id, JSON.stringify({ title: 'Amazing grace sample', channel: 'Choir' }), 'amazing grace sample choir', 1, Date.now()] });
await db.saveNote({ videoId: id, kind: 'lyrics', data: { lines: [{ t: 1, text: 'Amazing grace will lead me home' }] }, source: 'user', force: true });
expect((await app.request('/api/admin/lyrics/search?q=amazing')).status).toBe(401);
const headers = { 'x-admin': 'yes' };
let result = await (await app.request('/api/admin/lyrics/search?q=amazing+grace&hasChapters=no', { headers })).json();
expect(result.results[0]).toMatchObject({ videoId: id, title: 'Amazing grace sample', hasChapters: false });
expect(result.results[0].snippet).toContain('\u0001');
await db.saveNote({ videoId: id, kind: 'chapters', data: { items: [{ t: 1, title: 'Verse' }] }, source: 'user', force: true });
result = await (await app.request('/api/admin/lyrics/search?q=amazing&hasChapters=yes', { headers })).json();
expect(result.results[0].videoId).toBe(id);
await db.saveNote({ videoId: id, kind: 'lyrics', data: { lines: [{ t: 1, text: 'Bright morning star' }] }, source: 'user', force: true });
expect((await (await app.request('/api/admin/lyrics/search?q=amazing', { headers })).json()).total).toBe(0);
const rebuilt = await (await app.request('/api/admin/lyrics/search/rebuild', { method: 'POST', headers: { ...headers, 'content-type': 'application/json' }, body: '{}' })).json();
expect(rebuilt.ok).toBe(true);
expect(rebuilt.indexed).toBeGreaterThanOrEqual(1);
});
});

76
server/db.js
View File

@@ -20,6 +20,7 @@ const DB_PATH = process.env.DB_PATH || join(process.cwd(), 'data', 'ytplayer.db'
export const db = createClient({
url: 'file:' + DB_PATH,
});
let lyricsFtsAvailable = false;
// ---- Schema ----------------------------------------------------------------
@@ -279,6 +280,7 @@ export async function initDb() {
CREATE INDEX IF NOT EXISTS idx_audit_actor_action ON audit_log (actor, action, ts DESC);
`);
await migrateVideoMeta();
await initLyricsSearch();
try { await db.execute('ALTER TABLE media_cache ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0'); } catch { /* already migrated */ }
// Uploads shared from devices (PWA share target): who sent it, and whether it
// is listed in everyone's search (admin uploads) or reachable only by link.
@@ -307,6 +309,75 @@ export async function initDb() {
}
}
function lyricsSearchContent(value) {
try {
const data = typeof value === 'string' ? JSON.parse(value) : value;
return [...(data?.lines || []).flatMap(line => [line?.text, line?.secondary, line?.phonetic]), ...(data?.tags || [])]
.filter(Boolean).join(' ').slice(0, 500_000);
} catch { return ''; }
}
async function initLyricsSearch() {
await db.execute('CREATE TABLE IF NOT EXISTS lyrics_search_state (id INTEGER PRIMARY KEY CHECK(id=1), indexed_at INTEGER NOT NULL)');
try {
await db.execute("CREATE VIRTUAL TABLE IF NOT EXISTS lyrics_search_fts USING fts5(video_id UNINDEXED, content, tokenize='unicode61 remove_diacritics 2')");
lyricsFtsAvailable = true;
} catch {
await db.execute('CREATE TABLE IF NOT EXISTS lyrics_search_fts (video_id TEXT PRIMARY KEY, content TEXT NOT NULL)');
await db.execute('CREATE INDEX IF NOT EXISTS idx_lyrics_search_id ON lyrics_search_fts(video_id)');
lyricsFtsAvailable = false;
}
const state = await db.execute('SELECT id FROM lyrics_search_state WHERE id=1');
if (state.rows.length) return;
await db.execute('DELETE FROM lyrics_search_fts');
const rows = await db.execute("SELECT video_id,data FROM video_notes WHERE kind='lyrics'");
for (let i = 0; i < rows.rows.length; i += 100) {
const batch = rows.rows.slice(i, i + 100).map(row => ({
sql: lyricsFtsAvailable ? 'INSERT INTO lyrics_search_fts(video_id,content) VALUES(?,?)' : 'INSERT OR REPLACE INTO lyrics_search_fts(video_id,content) VALUES(?,?)',
args: [row.video_id, lyricsSearchContent(row.data)],
}));
if (batch.length) await db.batch(batch, 'write');
}
await db.execute('INSERT INTO lyrics_search_state(id,indexed_at) VALUES(1,unixepoch())');
}
export async function searchNoteLyrics(query, { limit = 30, offset = 0, hasChapters = false, withoutChapters = false } = {}) {
const words = String(query || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').match(/[\p{L}\p{N}_-]+/gu) || [];
if (!words.length) return [];
const boundedLimit = Math.max(1, Math.min(100, Number(limit) || 30));
const boundedOffset = Math.max(0, Math.min(1_000_000, Number(offset) || 0));
const chapterFilter = hasChapters ? "AND EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : withoutChapters ? "AND NOT EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : '';
if (lyricsFtsAvailable) {
const match = words.map(word => `"${word.replaceAll('"', '""')}"*`).join(' AND ');
const result = await db.execute({ sql: `SELECT video_id, snippet(lyrics_search_fts,1,char(1),char(2),' … ',18) AS snippet
FROM lyrics_search_fts WHERE lyrics_search_fts MATCH ? ${chapterFilter} ORDER BY bm25(lyrics_search_fts) LIMIT ? OFFSET ?`, args: [match, boundedLimit, boundedOffset] });
return result.rows.map(row => ({ videoId: row.video_id, snippet: row.snippet }));
}
const filters = words.map(() => 'content LIKE ?').join(' AND ');
const result = await db.execute({ sql: `SELECT video_id,substr(content,1,220) AS snippet FROM lyrics_search_fts WHERE ${filters} ORDER BY video_id LIMIT ? OFFSET ?`, args: [...words.map(word => `%${word}%`), boundedLimit, boundedOffset] });
return result.rows.map(row => ({ videoId: row.video_id, snippet: row.snippet }));
}
export async function countNoteLyrics(query, { hasChapters = false, withoutChapters = false } = {}) {
const words = String(query || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').match(/[\p{L}\p{N}_-]+/gu) || [];
if (!words.length) return 0;
const chapterFilter = hasChapters ? "AND EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : withoutChapters ? "AND NOT EXISTS (SELECT 1 FROM video_notes chapter WHERE chapter.video_id=lyrics_search_fts.video_id AND chapter.kind='chapters')" : '';
if (lyricsFtsAvailable) {
const match = words.map(word => `"${word.replaceAll('"', '""')}"*`).join(' AND ');
return Number((await db.execute({ sql: `SELECT COUNT(*) AS n FROM lyrics_search_fts WHERE lyrics_search_fts MATCH ? ${chapterFilter}`, args: [match] })).rows[0]?.n) || 0;
}
const filters = words.map(() => 'content LIKE ?').join(' AND ');
const result = await db.execute({ sql: `SELECT COUNT(*) AS n FROM lyrics_search_fts WHERE ${filters} ${chapterFilter}`, args: words.map(word => `%${word}%`) });
return Number(result.rows[0]?.n) || 0;
}
export async function rebuildLyricsSearch() {
await db.execute('DELETE FROM lyrics_search_fts');
await db.execute('DELETE FROM lyrics_search_state');
await initLyricsSearch();
return Number((await db.execute("SELECT COUNT(*) AS n FROM video_notes WHERE kind='lyrics'")).rows[0]?.n) || 0;
}
const YOUTUBE_ID = /^[\w-]{11}$/;
const folded = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
@@ -395,6 +466,11 @@ export async function saveNote({ videoId, kind, data, baseRev, source, updatedBy
VALUES (?, ?, ?, ?, ?, ?, unixepoch())`,
args: [videoId, kind, rev, json, source, updatedBy || null],
});
if (kind === 'lyrics') {
await tx.execute({ sql: 'DELETE FROM lyrics_search_fts WHERE video_id=?', args: [videoId] });
const content = lyricsSearchContent(data);
if (content) await tx.execute({ sql: 'INSERT INTO lyrics_search_fts(video_id,content) VALUES(?,?)', args: [videoId, content] });
}
await tx.commit();
return { ok: true, rev };
} catch (err) {