diff --git a/server/db.js b/server/db.js index 9fcc77f..f5e394c 100644 --- a/server/db.js +++ b/server/db.js @@ -257,6 +257,7 @@ export async function initDb() { last_used_at INTEGER ); `); + await migrateVideoMeta(); // Uploads shared from devices (PWA share target): who sent it, and whether it // is listed in everyone's search (admin uploads) or reachable only by link. for (const col of ['owner TEXT', 'listed INTEGER NOT NULL DEFAULT 1']) { @@ -284,6 +285,47 @@ export async function initDb() { } } +const YOUTUBE_ID = /^[\w-]{11}$/; +const folded = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase(); + +// These columns were added after video_meta shipped. Keep startup migration +// idempotent and page the backfill so a large catalogue never becomes one +// unbounded JS allocation. +export async function migrateVideoMeta({ pageSize = 500 } = {}) { + for (const [name, type] of [['url', 'TEXT'], ['channel_id', 'TEXT'], ['channel_url', 'TEXT'], ['published', 'TEXT'], ['lang', 'TEXT']]) { + try { await db.execute(`ALTER TABLE video_meta ADD COLUMN ${name} ${type}`); } catch { /* column already exists */ } + } + await db.execute('CREATE TABLE IF NOT EXISTS catalog_schema_migrations (version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL)'); + const done = await db.execute({ sql: 'SELECT 1 FROM catalog_schema_migrations WHERE version=1' }); + if (done.rows.length) return; + let cursor = ''; + for (;;) { + const rows = (await db.execute({ + sql: 'SELECT id,card,hay FROM video_meta WHERE id>? ORDER BY id LIMIT ?', args: [cursor, pageSize], + })).rows; + if (!rows.length) break; + const updates = []; + for (const row of rows) { + cursor = row.id; + let card = {}; + try { card = JSON.parse(row.card || '{}'); } catch { /* malformed legacy card */ } + const url = YOUTUBE_ID.test(row.id) ? `https://www.youtube.com/watch?v=${row.id}` : null; + const channelId = card.channelId || card.channel_id || null; + const channelUrl = card.channelUrl || card.channel_url || null; + const published = card.published || card.publishedAt || card.upload_date || null; + const lang = card.lang || card.language || null; + card = { ...card, url, channelId: channelId || '', channelUrl: channelUrl || '' }; + const hay = folded([card.title, card.channel, channelId, card.description, + ...(Array.isArray(card.tags) ? card.tags : []), ...(Array.isArray(card.categories) ? card.categories : [])].filter(Boolean).join(' ')); + updates.push({ sql: 'UPDATE video_meta SET card=?,url=?,channel_id=?,channel_url=?,published=?,lang=?,hay=? WHERE id=?', + args: [JSON.stringify(card), url, channelId, channelUrl, published, lang, hay || folded(row.hay), row.id] }); + } + await db.batch(updates, 'write'); + if (rows.length < pageSize) break; + } + await db.execute({ sql: 'INSERT OR IGNORE INTO catalog_schema_migrations(version,applied_at) VALUES(1,?)', args: [Date.now()] }); +} + // ---- Shared video notes (lyrics / chapters) --------------------------------- function noteRow(r) { diff --git a/server/innertube.js b/server/innertube.js index c6ae686..356479c 100644 --- a/server/innertube.js +++ b/server/innertube.js @@ -20,6 +20,7 @@ function cardsFromSections(sections) { const path = be.canonicalBaseUrl || (be.browseId ? `/channel/${be.browseId}` : ''); out.push({ id: v.videoId, + url: `https://www.youtube.com/watch?v=${v.videoId}`, title: v.title?.runs?.map((r) => r.text).join('') || '(untitled)', channel: owner.text || '', channelId: be.browseId || '', diff --git a/server/search-cache.js b/server/search-cache.js index 423ec32..934ad35 100644 --- a/server/search-cache.js +++ b/server/search-cache.js @@ -2,7 +2,7 @@ // repeat query — from any device, or after a restart — never waits on YouTube. // Capped by row count (default 250,000 queries) AND bytes (default 2 GiB), LRU. import { db } from './db.js'; -import { ingest, withLocalThumbnails, trimCatalog } from './video-catalog.js'; +import { ingest, withLocalThumbnails, trimCatalog, foldSearchText } from './video-catalog.js'; export const MAX_ROWS = Number(process.env.SEARCH_CACHE_MAX_ROWS) || 250_000; export const MAX_BYTES = Number(process.env.SEARCH_CACHE_MAX_BYTES) || 2 * 1024 ** 3; @@ -70,15 +70,28 @@ export async function rememberVideos(cards) { // Cards whose title/channel contain EVERY word of the query, most-seen first. export async function searchVideos(q, limit = 60) { - const words = String(q).toLowerCase().split(/\s+/).filter(Boolean).slice(0, 8); + const folded = foldSearchText(q).trim(); + const words = folded.split(/\s+/).filter(Boolean).slice(0, 8); if (!words.length) return []; const like = (w) => '%' + w.replace(/[\\%_]/g, '\\$&') + '%'; const r = await db.execute({ - sql: `SELECT card FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')} + sql: `SELECT card,seen FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')} ORDER BY seen DESC, updated_at DESC LIMIT ?`, - args: [...words.map(like), limit], + args: [...words.map(like), Math.max(limit * 10, 200)], }); - return withLocalThumbnails(r.rows.map((x) => { try { return JSON.parse(x.card); } catch { return null; } }).filter(Boolean)); + const titleHasAll = title => words.every(word => foldSearchText(title).includes(word)); + const phrase = folded; + const ranked = r.rows.map(row => { + try { + const card = JSON.parse(row.card); + const title = foldSearchText(card.title); + const channel = foldSearchText(card.channel); + const score = (title === phrase || title.includes(phrase) ? 4 : 0) + + (titleHasAll(card.title) ? 2 : 0) + (words.some(word => channel.includes(word)) ? 1 : 0); + return { card: { ...card, url: card.url || `https://www.youtube.com/watch?v=${card.id}` }, score, seen: Number(row.seen) || 0 }; + } catch { return null; } + }).filter(Boolean).sort((a, b) => b.score - a.score || b.seen - a.seen).slice(0, limit).map(x => x.card); + return withLocalThumbnails(ranked); } export async function trimVideos(max = MAX_VIDEOS) { diff --git a/server/search-cache.test.js b/server/search-cache.test.js index 47a4033..7d25f50 100644 --- a/server/search-cache.test.js +++ b/server/search-cache.test.js @@ -21,11 +21,11 @@ test('trim evicts least-recently-used rows over the row cap', async () => { test('known videos are remembered and found by every word', async () => { await sc.rememberVideos([ - { id: 'a1', title: 'Way Maker - Sinach', channel: 'Worship' }, - { id: 'b2', title: 'Way Maker (live)', channel: 'Other' }, - { id: 'c3', title: 'Goodness of God', channel: 'Worship' }, + { id: 'aaa11111111', title: 'Way Maker - Sinach', channel: 'Worship' }, + { id: 'bbb22222222', title: 'Way Maker (live)', channel: 'Other' }, + { id: 'ccc33333333', title: 'Goodness of God', channel: 'Worship' }, ]); - expect((await sc.searchVideos('way maker')).map((c) => c.id).sort()).toEqual(['a1', 'b2']); - expect((await sc.searchVideos('maker worship')).map((c) => c.id)).toEqual(['a1']); + expect((await sc.searchVideos('way maker')).map((c) => c.id).sort()).toEqual(['aaa11111111', 'bbb22222222']); + expect((await sc.searchVideos('maker worship')).map((c) => c.id)).toEqual(['aaa11111111']); expect(await sc.searchVideos('100%')).toEqual([]); }); diff --git a/server/server.js b/server/server.js index 771d7dc..7989f36 100644 --- a/server/server.js +++ b/server/server.js @@ -268,6 +268,7 @@ function slimEntry(j) { return { id, title: pick(j, 'title') || '(untitled)', + url: /^[\w-]{11}$/.test(id) ? `https://www.youtube.com/watch?v=${id}` : null, channel: pickChannel(j), channelId: pickChannelId(j), channelUrl: pickChannelUrl(j), @@ -418,6 +419,8 @@ const searchLibrary = async (q) => { // This server's own library first — it still answers when YouTube is unreachable. try { return (await notesDb.listUploads({ q, limit: 20, listedOnly: true })).map(uploads.card); } catch { return []; } }; +const withYoutubeUrl = card => card && /^[\w-]{11}$/.test(card.id || '') + ? { ...card, url: `https://www.youtube.com/watch?v=${card.id}` } : card; // GET /api/search/local?q= — videos this server already knows (from every // earlier search) plus its own uploads; answers in milliseconds so the app can @@ -426,7 +429,7 @@ app.get('/api/search/local', async (c) => { const q = (c.req.query('q') || '').trim(); if (!q) return c.json({ ok: false, error: 'empty query' }, 400); const [mine, known] = await Promise.all([searchLibrary(q), searchCacheDb.searchVideos(q).catch(() => [])]); - return c.json({ ok: true, results: [...mine, ...known], local: true }); + return c.json({ ok: true, results: [...mine, ...known].map(withYoutubeUrl), local: true }); }); // GET /api/search?q=[&refresh=1] @@ -439,7 +442,7 @@ app.get('/api/search', async (c) => { if (!q) return c.json({ ok: false, error: 'empty query' }, 400); const key = q.toLowerCase(); const mine = await searchLibrary(q); - const answer = (yt, extra = {}) => c.json({ ok: true, results: [...mine, ...yt], ...extra }); + const answer = (yt, extra = {}) => c.json({ ok: true, results: [...mine, ...yt].map(withYoutubeUrl), ...extra }); if (c.req.query('refresh') !== '1') { let hit = searchCache.get(key); if (!hit) { diff --git a/server/video-catalog.js b/server/video-catalog.js index 081af55..902eceb 100644 --- a/server/video-catalog.js +++ b/server/video-catalog.js @@ -7,6 +7,8 @@ const SOURCES = new Set(['search', 'search-cache', 'client-search', 'channel', ' const text = (v, n = 300) => typeof v === 'string' ? v.trim().slice(0, n) : ''; const positive = v => Number.isFinite(Number(v)) && Number(v) > 0 ? Number(v) : 0; const canonicalThumb = id => `https://i.ytimg.com/vi/${id}/hqdefault.jpg`; +const canonicalUrl = id => `https://www.youtube.com/watch?v=${id}`; +export const foldSearchText = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase(); export const channelKey = c => String(c.channelId || c.channel || '').toLowerCase().replace(/\s*-\s*topic$|vevo$|\s+official$/g, '').trim(); function safeThumbnail(url) { try { const u = new URL(url); return u.protocol === 'https:' && /^(?:i|i\d)\.ytimg\.com$/.test(u.hostname) && !u.port && !u.username && !u.password; } @@ -23,14 +25,16 @@ export function normalizeCard(v) { // Never fetch arbitrary client URLs (including redirects or private hosts). if (safeThumbnail(u.href)) thumbnail = u.href; } catch { /* canonical YouTube art */ } - const card = { id: v.id, title, channel: text(v.channel), channelId: text(v.channelId, 64), + const card = { id: v.id, title, url: canonicalUrl(v.id), channel: text(v.channel), channelId: text(v.channelId ?? v.channel_id, 64), duration: positive(v.duration), thumbnail }; try { - const u = new URL(v.channelUrl); + const u = new URL(v.channelUrl || v.channel_url); if (u.protocol === 'https:' && /^(www\.)?youtube\.com$/.test(u.hostname)) card.channelUrl = u.href; } catch { /* optional */ } for (const key of ['tags', 'categories']) if (Array.isArray(v[key])) card[key] = v[key].slice(0, 30).map(x => text(x, 80)).filter(Boolean); if (v.description) card.description = text(v.description, 1200); + if (v.published || v.publishedAt || v.upload_date) card.published = text(v.published || v.publishedAt || v.upload_date, 40); + if (v.lang || v.language) card.lang = text(v.lang || v.language, 24); if (positive(v.viewCount ?? v.view_count)) card.viewCount = positive(v.viewCount ?? v.view_count); return card; } @@ -67,9 +71,12 @@ async function ingestBatch(cards, source, fillMissing) { await db.batch(batch.flatMap(c => { // Sparse playlist cards must not erase richer extractor metadata. const merged = fillMissing ? mergeMissing(old.get(c.id) || {}, c) : { ...(old.get(c.id) || {}), ...Object.fromEntries(Object.entries(c).filter(([, v]) => v !== '' && v !== 0 && (!Array.isArray(v) || v.length))) }; - return [{ sql: `INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,1,?) - ON CONFLICT(id) DO UPDATE SET card=excluded.card,hay=excluded.hay,seen=seen+1,updated_at=excluded.updated_at`, - args: [c.id, JSON.stringify(merged), `${merged.title} ${merged.channel || ''} ${(merged.tags || []).join(' ')}`.toLowerCase(), now] }, + const hay = foldSearchText([merged.title, merged.channel, merged.channelId, merged.description, + ...(merged.tags || []), ...(merged.categories || [])].filter(Boolean).join(' ')); + return [{ sql: `INSERT INTO video_meta (id,card,hay,seen,updated_at,url,channel_id,channel_url,published,lang) VALUES (?,?,?,1,?,?,?,?,?,?) + ON CONFLICT(id) DO UPDATE SET card=excluded.card,hay=excluded.hay,seen=seen+1,updated_at=excluded.updated_at, + url=excluded.url,channel_id=excluded.channel_id,channel_url=excluded.channel_url,published=excluded.published,lang=excluded.lang`, + args: [c.id, JSON.stringify(merged), hay, now, merged.url, merged.channelId || null, merged.channelUrl || null, merged.published || null, merged.lang || null] }, { sql: `INSERT INTO video_meta_sources (video_id,source,last_seen) VALUES (?,?,?) ON CONFLICT(video_id,source) DO UPDATE SET discoveries=discoveries+1,last_seen=excluded.last_seen`, args: [c.id, source, now] }, { sql: `INSERT INTO video_thumbnails (video_id,url) VALUES (?,?) ON CONFLICT(video_id) DO UPDATE SET retry_at=0 diff --git a/server/video-catalog.test.js b/server/video-catalog.test.js new file mode 100644 index 0000000..eb0c205 --- /dev/null +++ b/server/video-catalog.test.js @@ -0,0 +1,53 @@ +import { test, expect } from 'bun:test'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const root = mkdtempSync(join(tmpdir(), 'ytp-video-catalog-')); +process.env.DB_PATH = join(root, 'test.db'); +const db = await import('./db.js'); +const catalog = await import('./video-catalog.js'); +await db.initDb(); + +test('catalog migration backfills canonical URLs and rich metadata, excluding upload ids', async () => { + await db.db.execute('DELETE FROM catalog_schema_migrations WHERE version=1'); + await db.db.execute({ sql: 'INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,?,?)', + args: ['abcDEF12345', JSON.stringify({ id: 'abcDEF12345', title: 'Café Worship', channel: 'Choir', channelId: 'UC123', channelUrl: 'https://www.youtube.com/@choir', tags: ['live'], publishedAt: '2024-01-01', language: 'en' }), 'cafe worship choir', 1, 1] }); + await db.db.execute({ sql: 'INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,?,?)', + args: ['upl_abcdef12', JSON.stringify({ id: 'upl_abcdef12', title: 'Upload' }), 'upload', 1, 1] }); + await db.migrateVideoMeta({ pageSize: 1 }); + await db.migrateVideoMeta({ pageSize: 1 }); + const youtube = (await db.db.execute("SELECT * FROM video_meta WHERE id='abcDEF12345'")).rows[0]; + const upload = (await db.db.execute("SELECT * FROM video_meta WHERE id='upl_abcdef12'")).rows[0]; + expect(youtube.url).toBe('https://www.youtube.com/watch?v=abcDEF12345'); + expect(youtube.channel_id).toBe('UC123'); + expect(youtube.channel_url).toBe('https://www.youtube.com/@choir'); + expect(youtube.published).toBe('2024-01-01'); + expect(youtube.lang).toBe('en'); + expect(youtube.hay).toContain('cafe'); + expect(JSON.parse(youtube.card).url).toBe(youtube.url); + expect(upload.url).toBeNull(); +}); + +test('ingest stores canonical source URL and channel metadata on the card and row', async () => { + await catalog.ingest([{ id: 'xyzABC12345', title: 'Música', channel: 'Choir', channel_id: 'UC456', channel_url: 'https://www.youtube.com/@choir', tags: ['Praise'], description: 'Sunday service' }], 'deep'); + const row = (await db.db.execute("SELECT * FROM video_meta WHERE id='xyzABC12345'")).rows[0]; + const card = JSON.parse(row.card); + expect(row.url).toBe('https://www.youtube.com/watch?v=xyzABC12345'); + expect(row.channel_id).toBe('UC456'); + expect(card.url).toBe(row.url); + expect(row.hay).toContain('musica'); + expect(row.hay).toContain('praise'); +}); + +test('accent-insensitive matching ranks title phrase ahead of channel match', async () => { + await catalog.ingest([ + { id: 'phrase12345', title: 'Café Worship Live', channel: 'Other' }, + { id: 'channel1234', title: 'Sunday Set', channel: 'Cafe Worship Live' }, + { id: 'words123456', title: 'Worship in the Cafe', channel: 'Other' }, + ], 'deep'); + const search = await import('./search-cache.js'); + const results = await search.searchVideos('cafe worship live'); + expect(results[0].id).toBe('phrase12345'); + expect(results[0].url).toBe('https://www.youtube.com/watch?v=phrase12345'); +});