diff --git a/frontend/admin.html b/frontend/admin.html index 8ff0ec5..5743cfa 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -342,6 +342,14 @@ .collection-job p { margin: 4px 0; color: var(--text-2); } #metadataDetail { white-space: pre-wrap; overflow-wrap: anywhere; max-height: 400px; overflow: auto; font-size: 13px; } @media (max-width: 480px) { .analytics-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } .metric { padding: 12px; } .metric b { font-size: 22px; } } + .deep-grid { display: grid; grid-template-columns: repeat(auto-fit,minmax(min(100%,240px),1fr)); gap: 12px; } + .deep-grid label { display: grid; gap: 5px; min-width: 0; color: var(--text); font-size: 13px; } + .deep-search .muted { color: var(--text-2); } + .deep-grid .field { width: 100%; flex: initial; } + .deep-grid textarea.field { height: auto; min-height: 110px; padding: 10px 12px; resize: vertical; } + .deep-job { padding: 14px 0; border-bottom: 1px solid var(--line-2); overflow-wrap: anywhere; } + .deep-job progress { width: 100%; height: 8px; accent-color: var(--a1); } + .deep-log { max-height: 240px; overflow: auto; white-space: pre-wrap; font: 12px/1.5 var(--mono); color: var(--text-2); } @media (min-width: 761px) and (max-width: 1100px) { .top { flex-wrap: wrap; } .tabs { order: 2; width: 100%; margin-left: 0; overflow-x: auto; } @@ -628,6 +636,29 @@

Most played

+ +

API access

Tokens let scripts write shared lyrics and chapters. A token is shown once — store it in your secrets vault.

@@ -683,9 +714,10 @@ peers: '', history: '', analytics: '', + 'deep-search': '', access: '', }; - const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -705,6 +737,7 @@ if (name === 'library') loadUploads(); if (name === 'access') loadTokens(); if (name === 'analytics') loadAnalytics(); + if (name === 'deep-search') loadDeepSearch(); window.scrollTo({ top: 0 }); } function setBadge(n) { @@ -798,6 +831,52 @@ $('metadataFilterForm').addEventListener('submit', e => { e.preventDefault(); metadataOffset = 0; loadMetadata(); }); $('metadataPrev').addEventListener('click', () => { metadataOffset = Math.max(0, metadataOffset - 50); loadMetadata(); }); $('metadataNext').addEventListener('click', () => { metadataOffset += 50; loadMetadata(); }); + + /* ---------------- durable deep search ---------------- */ + let deepTimer = null; + const lines = id => $(id).value.split(/\r?\n/).map(s => s.trim()).filter(Boolean); + function deepBody() { + return { seeds: lines('deepSeeds'), modifiers: lines('deepModifiers'), maxQueries: Number($('deepMaxQueries').value), + resultsPerQuery: Number($('deepResults').value), relatedDepth: Number($('deepDepth').value), relatedTop: Number($('deepRelatedTop').value), + channels: lines('deepChannels'), playlists: lines('deepPlaylists'), delaySeconds: Number($('deepDelay').value), dailyCap: Number($('deepDailyCap').value) }; + } + if (!$('deepSeeds').value) $('deepSeeds').value = 'worship\npraise\nhymns\ngospel\npagsamba\npapuri\nmga awit ng papuri\nebanghelyo\ndayeg\npagdayeg\nmga alawiton\nmaayong balita\nadoración\nalabanza\nhimnos\nevangelio\nHillsong Worship\nElevation Worship\nBethel Music\nPlanetshakers\nDon Moen\nSandi Patty'; + if (!$('deepModifiers').value) $('deepModifiers').value = 'lyrics\nlive\nacoustic\ninstrumental\nkaraoke'; + $('deepEstimate').addEventListener('click', async () => { + const label = $('deepEstimateText'); label.textContent = 'Counting the plan…'; + try { const j = await api('/api/admin/deep-search/estimate', { method: 'POST', body: deepBody() }); + label.textContent = j.ok ? `${j.queries} unique queries · ${j.browseSources} browse sources · about ${j.estimatedMinutes} min · ${j.indexSize.toLocaleString()} indexed · ${j.dailyRemaining} daily requests remain` : j.error || 'Estimate unavailable'; + } catch { label.textContent = 'Could not reach the server.'; } + }); + $('deepSearchForm').addEventListener('submit', async e => { + e.preventDefault(); const button = $('deepStart'); button.disabled = true; + try { const j = await api('/api/admin/deep-search/start', { method: 'POST', body: deepBody() }); + $('deepEstimateText').textContent = j.ok ? 'Job queued. It will continue if this page closes.' : j.error || 'Could not start job.'; + await loadDeepSearch(); + } catch { $('deepEstimateText').textContent = 'Could not reach the server.'; } + finally { button.disabled = false; } + }); + async function loadDeepSearch() { + try { + const j = await api('/api/admin/deep-search/jobs'); if (!j.ok) throw Error(j.error || 'Could not load deep-search jobs.'); + $('deepIndexStatus').textContent = `${Number(j.indexSize).toLocaleString()} videos in the local search index.`; + $('deepJobs').innerHTML = j.jobs.length ? j.jobs.map(job => { + const active = ['queued','running','paused'].includes(job.status), progress = job.queriesTotal ? Math.round(job.queriesDone / job.queriesTotal * 100) : 0; + const eta = job.etaMs == null ? 'estimating' : job.etaMs < 60000 ? `${Math.ceil(job.etaMs / 1000)} sec` : `${Math.ceil(job.etaMs / 60000)} min`; + const controls = job.status === 'running' || job.status === 'queued' ? `` : job.status === 'paused' ? `` : ''; + return `
${esc(job.status[0].toUpperCase() + job.status.slice(1))}${job.queriesDone}/${job.queriesTotal} queries · ${job.newVideos} new videos · ETA ${eta}${controls}${active ? `` : ''}
${job.current ? `

Current: ${esc(job.current)}

` : ''}${job.errors?.length ? `
${job.errors.length} errors${job.errors.map(x => `

${esc(x.query)}: ${esc(x.error)}

`).join('')}
` : ''}
Live log (${job.log?.length || 0})
${(job.log || []).map(x => `${new Date(x.at).toLocaleTimeString()} ${esc(x.message)}`).join('\n')}
`; + }).join('') : '

No deep-search runs yet.

'; + clearTimeout(deepTimer); deepTimer = null; + if (tab === 'deep-search' && j.jobs.some(job => ['queued','running'].includes(job.status))) deepTimer = setTimeout(loadDeepSearch, 3000); + } catch (error) { $('deepIndexStatus').textContent = error.message || 'Could not load deep-search progress.'; } + } + $('deepRefresh').addEventListener('click', loadDeepSearch); + $('deepJobs').addEventListener('click', async e => { + const button = e.target.closest('[data-deep-action]'); if (!button) return; + button.disabled = true; + try { const j = await api(`/api/admin/deep-search/${encodeURIComponent(button.dataset.id)}/${button.dataset.deepAction}`, { method: 'POST', body: {} }); if (!j.ok) $('deepIndexStatus').textContent = j.error || `Could not ${button.dataset.deepAction} job.`; } + finally { await loadDeepSearch(); } + }); $('metadataTable').addEventListener('click', async e => { const btn = e.target.closest('[data-metadata-id]'); if (!btn) return; const j = await api('/api/admin/metadata/' + btn.dataset.metadataId); if (!j.ok) return; @@ -977,7 +1056,7 @@ const [m, u] = await Promise.all([api('/api/admin/media'), api('/api/uploads?limit=200')]); const seen = new Set(); SONGS = []; - for (const r of (m.media || [])) { seen.add(r.id); SONGS.push({ id: r.id, title: r.title || r.id, sub: r.channel, lines: r.lyricsLines || 0 }); } + for (const r of (m.media || [])) { seen.add(r.id); SONGS.push({ id: r.id, title: r.title || r.id, sub: r.channel, lines: r.lyricsLines || 0, sourceUrl: r.sourceUrl }); } for (const r of (u.uploads || [])) if (!seen.has(r.id)) SONGS.push({ id: r.id, title: r.title, sub: r.artist || 'Upload', lines: -1 }); paintSongs(); } @@ -987,6 +1066,7 @@ $('songList').innerHTML = rows.length ? rows.map((r) => `
+ ${r.sourceUrl ? `Source` : ''}
`).join('') : '
Nothing matches — paste a link above to open any YouTube song.
'; paintWhisper(); @@ -994,8 +1074,17 @@ $('songFilter').addEventListener('input', paintSongs); $('songList').addEventListener('click', (e) => { const whisper = e.target.closest('[data-whisper]'); if (whisper) { startWhisper(whisper.dataset.whisper); return; } + const source = e.target.closest('[data-redownload]'); if (source) { startSourceRedownload(source); return; } const b = e.target.closest('[data-open]'); if (b) leOpen(b.dataset.open); }); + async function startSourceRedownload(button) { + button.disabled = true; const original = button.textContent; button.textContent = 'Queueing…'; + try { const result = await api(`/api/admin/media/${encodeURIComponent(button.dataset.redownload)}/redownload`, { method: 'POST', body: {} }); + button.textContent = result.ok ? 'Queued' : 'Could not queue'; + button.title = result.error || 'The server-owned download continues if this page closes.'; + } catch { button.textContent = 'Connection failed'; } + setTimeout(() => { button.textContent = original; button.disabled = false; }, 2000); + } $('recentChips').addEventListener('click', (e) => { const b = e.target.closest('[data-open]'); if (b) leOpen(b.dataset.open); }); /* ---------------- p2p ---------------- */ diff --git a/playwright.admin-deep-search.config.js b/playwright.admin-deep-search.config.js new file mode 100644 index 0000000..671b668 --- /dev/null +++ b/playwright.admin-deep-search.config.js @@ -0,0 +1,3 @@ +const { defineConfig } = require('@playwright/test'); +const classic = require('./playwright.classic.config'); +module.exports = defineConfig({ ...classic, testMatch: /admin-deep-search\.spec\.js/ }); diff --git a/server/db.js b/server/db.js index 9fcc77f..a3f8e4e 100644 --- a/server/db.js +++ b/server/db.js @@ -168,6 +168,15 @@ export async function initDb() { ); CREATE INDEX IF NOT EXISTS idx_collections_queue ON metadata_collections (status,created_at); + CREATE TABLE IF NOT EXISTS deep_search_jobs ( + id TEXT PRIMARY KEY, status TEXT NOT NULL, plan TEXT NOT NULL, options TEXT NOT NULL, + state TEXT NOT NULL DEFAULT '{}', error TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_deep_search_jobs ON deep_search_jobs(status,created_at); + CREATE TABLE IF NOT EXISTS deep_search_daily ( + day TEXT PRIMARY KEY, queries INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE IF NOT EXISTS lyric_transcriptions ( id TEXT PRIMARY KEY, video_id TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'queued', base_rev INTEGER NOT NULL DEFAULT 0, result TEXT, error TEXT, stage TEXT NOT NULL DEFAULT 'queued', @@ -257,6 +266,7 @@ export async function initDb() { last_used_at INTEGER ); `); + await migrateVideoMeta(); // Uploads shared from devices (PWA share target): who sent it, and whether it // is listed in everyone's search (admin uploads) or reachable only by link. for (const col of ['owner TEXT', 'listed INTEGER NOT NULL DEFAULT 1']) { @@ -284,6 +294,47 @@ export async function initDb() { } } +const YOUTUBE_ID = /^[\w-]{11}$/; +const folded = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase(); + +// These columns were added after video_meta shipped. Keep startup migration +// idempotent and page the backfill so a large catalogue never becomes one +// unbounded JS allocation. +export async function migrateVideoMeta({ pageSize = 500 } = {}) { + for (const [name, type] of [['url', 'TEXT'], ['channel_id', 'TEXT'], ['channel_url', 'TEXT'], ['published', 'TEXT'], ['lang', 'TEXT']]) { + try { await db.execute(`ALTER TABLE video_meta ADD COLUMN ${name} ${type}`); } catch { /* column already exists */ } + } + await db.execute('CREATE TABLE IF NOT EXISTS catalog_schema_migrations (version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL)'); + const done = await db.execute({ sql: 'SELECT 1 FROM catalog_schema_migrations WHERE version=1' }); + if (done.rows.length) return; + let cursor = ''; + for (;;) { + const rows = (await db.execute({ + sql: 'SELECT id,card,hay FROM video_meta WHERE id>? ORDER BY id LIMIT ?', args: [cursor, pageSize], + })).rows; + if (!rows.length) break; + const updates = []; + for (const row of rows) { + cursor = row.id; + let card = {}; + try { card = JSON.parse(row.card || '{}'); } catch { /* malformed legacy card */ } + const url = YOUTUBE_ID.test(row.id) ? `https://www.youtube.com/watch?v=${row.id}` : null; + const channelId = card.channelId || card.channel_id || null; + const channelUrl = card.channelUrl || card.channel_url || null; + const published = card.published || card.publishedAt || card.upload_date || null; + const lang = card.lang || card.language || null; + card = { ...card, url, channelId: channelId || '', channelUrl: channelUrl || '' }; + const hay = folded([card.title, card.channel, channelId, card.description, + ...(Array.isArray(card.tags) ? card.tags : []), ...(Array.isArray(card.categories) ? card.categories : [])].filter(Boolean).join(' ')); + updates.push({ sql: 'UPDATE video_meta SET card=?,url=?,channel_id=?,channel_url=?,published=?,lang=?,hay=? WHERE id=?', + args: [JSON.stringify(card), url, channelId, channelUrl, published, lang, hay || folded(row.hay), row.id] }); + } + await db.batch(updates, 'write'); + if (rows.length < pageSize) break; + } + await db.execute({ sql: 'INSERT OR IGNORE INTO catalog_schema_migrations(version,applied_at) VALUES(1,?)', args: [Date.now()] }); +} + // ---- Shared video notes (lyrics / chapters) --------------------------------- function noteRow(r) { diff --git a/server/deep-search.js b/server/deep-search.js new file mode 100644 index 0000000..5e387d7 --- /dev/null +++ b/server/deep-search.js @@ -0,0 +1,214 @@ +import { randomUUID } from 'node:crypto'; +import { db as defaultDb } from './db.js'; +import * as catalog from './video-catalog.js'; +import * as innertube from './innertube.js'; +import * as searchCache from './search-cache.js'; + +const MAX_SEEDS = 200, MAX_PLAN = 500, MAX_ERRORS = 50; +const clean = value => String(value || '').replace(/[\u0000-\u001f\u007f]/g, ' ').trim().slice(0, 200); +const fold = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase().replace(/\s+/g, ' ').trim(); +const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } }; +const sleepDefault = ms => new Promise(resolve => setTimeout(resolve, ms)); + +export const DEFAULT_PRESETS = [ + 'worship', 'praise', 'hymns', 'gospel', 'pagsamba', 'papuri', 'mga awit ng papuri', 'ebanghelyo', + 'dayeg', 'pagdayeg', 'mga alawiton', 'maayong balita', 'adoración', 'alabanza', 'himnos', 'evangelio', + 'Hillsong Worship', 'Elevation Worship', 'Bethel Music', 'Planetshakers', 'Don Moen', 'Sandi Patty', +]; +export const DEFAULT_MODIFIERS = ['lyrics', 'live', 'acoustic', 'instrumental', 'karaoke']; + +export function expandQueryPlan({ seeds = [], modifiers = [], maxQueries = 80 } = {}) { + const seedList = (Array.isArray(seeds) ? seeds : String(seeds).split(/\r?\n/)).map(clean).filter(Boolean).slice(0, MAX_SEEDS); + const mods = (Array.isArray(modifiers) ? modifiers : String(modifiers).split(/[,\r\n]+/)).map(clean).filter(Boolean).slice(0, 30); + const cap = Math.max(1, Math.min(MAX_PLAN, Number(maxQueries) || 1)); + const out = [], seen = new Set(); + for (const seed of seedList) for (const modifier of ['', ...mods]) { + const query = clean(modifier ? `${seed} ${modifier}` : seed), key = fold(query); + if (key && !seen.has(key)) { seen.add(key); out.push(query); if (out.length >= cap) return out; } + } + return out; +} + +function publicJob(row, indexSize = 0) { + const state = parse(row.state); + const done = Number(state.queriesDone) || 0, total = JSON.parse(row.plan || '[]').length; + const elapsed = Math.max(0, Date.now() - Number(row.created_at)); + const etaMs = done ? Math.max(0, Math.round(elapsed / done * (total - done))) : null; + return { id: row.id, status: row.status, plan: JSON.parse(row.plan || '[]'), options: parse(row.options), + queriesDone: done, queriesTotal: total, current: state.current || '', newVideos: Number(state.newVideos) || 0, + totalIndexSize: indexSize, errors: state.errors || [], log: state.log || [], etaMs, + createdAt: Number(row.created_at), updatedAt: Number(row.updated_at), error: row.error || null }; +} + +export function createDeepSearchService({ db = defaultDb, searchDeep = innertube.searchDeep, related = innertube.related, + browseUploads = innertube.browseUploads, ingest = catalog.ingest, putSearch = searchCache.put, + sleep = sleepDefault, random = Math.random, now = Date.now, autoStart = true } = {}) { + let worker = null; + let timer = null; + const log = (state, message) => { state.log ||= []; state.log.push({ at: now(), message: String(message).slice(0, 240) }); if (state.log.length > 100) state.log.splice(0, state.log.length - 100); }; + async function indexSize() { return Number((await db.execute('SELECT COUNT(*) AS n FROM video_meta')).rows[0]?.n || 0); } + async function getJob(id) { return (await db.execute({ sql: 'SELECT * FROM deep_search_jobs WHERE id=?', args: [id] })).rows[0] || null; } + async function checkpoint(job, state, status = 'running', error = null) { + const r = await db.execute({ sql: "UPDATE deep_search_jobs SET state=?,status=?,error=?,updated_at=? WHERE id=? AND status='running'", args: [JSON.stringify(state), status, error, now(), job.id] }); + if (!r.rowsAffected) { const stop = new Error('Deep search paused or cancelled.'); stop.name = 'DeepSearchStopped'; throw stop; } + } + async function reserveDaily(cap) { + const day = new Date(now()).toISOString().slice(0, 10); + const r = await db.execute({ sql: `INSERT INTO deep_search_daily(day,queries) VALUES(?,1) + ON CONFLICT(day) DO UPDATE SET queries=queries+1 WHERE queries c?.id).map(c => [c.id, c])).values()]; + if (unique.length) { + const validIds = unique.filter(c => /^[\w-]{11}$/.test(c.id)).map(c => c.id); + const existing = validIds.length ? (await db.execute({ sql: `SELECT id FROM video_meta WHERE id IN (${validIds.map(() => '?').join(',')})`, args: validIds })).rows : []; + const present = new Set(existing.map(row => row.id)); + await ingest(unique, 'deep'); + state.newVideos += validIds.filter(id => !present.has(id)).length; + } + await putSearch(query, (cacheCards || unique).slice(0, 200)); + } + async function requireDailyBudget() { + const used = await reserveDaily(Math.max(1, Number(options.dailyCap) || 1000)); + if (used !== null) return; + state.dailyPausedDay = new Date(now()).toISOString().slice(0, 10); + log(state, 'Daily query cap reached; job will resume tomorrow.'); + await checkpoint(job, state, 'paused'); + const stop = new Error('Daily cap reached.'); stop.name = 'DeepSearchStopped'; throw stop; + } + try { + for (let qi = state.queriesDone; qi < plan.length; qi++) { + const query = plan[qi]; + if (state.current !== query) state.currentIds = []; + state.current = query; await checkpoint(job, state); + await requireDailyBudget(); + await checkpoint(job, state); + let cards, lastError; + for (let attempt = 0; attempt < 3; attempt++) { + try { + cards = await searchDeep(query, { limit: Math.max(1, Math.min(200, Number(options.resultsPerQuery) || 50)), maxPages: 14, timeoutMs: 30000, + pageDelayMs: Math.max(0, Number(options.delayMs) || 0), jitterMs: Math.max(0, Number(options.jitterMs) || 0), + onPage: async pageCards => { + await checkpoint(job, state); + const saved = new Set(state.currentIds || []); + const fresh = pageCards.filter(card => !saved.has(card.id)); + await saveCards(query, fresh, pageCards); + state.currentIds = pageCards.map(card => card.id); + await checkpoint(job, state); + } }); + await checkpoint(job, state); + lastError = null; break; + } catch (error) { if (error.name === 'DeepSearchStopped') throw error; lastError = error; if (attempt < 2) await sleep(Math.min(60000, (Number(options.delayMs) || 10000) * 2 ** attempt + Math.floor(random() * 3000))); } + } + if (lastError) { state.errors.push({ query, error: String(lastError.message).slice(0, 200) }); if (state.errors.length > MAX_ERRORS) state.errors.shift(); log(state, `Search failed: ${query}`); } + else { + const saved = new Set(state.currentIds || []); + const remaining = (cards || []).filter(card => !saved.has(card.id)); + await saveCards(query, remaining, cards || []); + state.currentIds = (cards || []).map(card => card.id); + const depth = Math.max(0, Math.min(2, Number(options.relatedDepth) || 0)); + if (depth) { + let frontier = (cards || []).slice(0, Math.max(1, Math.min(10, Number(options.relatedTop) || 3))).map(c => ({ id: c.id, depth: 1 })); + for (let level = 1; level <= depth && frontier.length; level++) { + const next = []; + for (const item of frontier) { + if (state.relatedSeen.includes(item.id)) continue; + state.relatedSeen.push(item.id); + try { + await sleep(Math.max(0, Number(options.delayMs) || 10000) + Math.floor(random() * (Number(options.jitterMs) || 3000))); + await checkpoint(job, state); + await requireDailyBudget(); + const recs = await related(item.id); await checkpoint(job, state); + await saveCards(`${query} related ${item.id}`, recs); + if (level < depth) next.push(...recs.slice(0, 2).map(c => ({ id: c.id, depth: level + 1 }))); + } catch (error) { if (error.name === 'DeepSearchStopped') throw error; state.errors.push({ query: `related:${item.id}`, error: String(error.message).slice(0, 200) }); } + } + frontier = next; + } + } + log(state, `Saved ${cards?.length || 0} results for “${query}”.`); + } + state.queriesDone = qi + 1; state.current = ''; state.currentIds = []; await checkpoint(job, state); + if (qi + 1 < plan.length) await sleep(Math.max(0, Number(options.delayMs) || 10000) + Math.floor(random() * (Number(options.jitterMs) || 3000))); + } + for (const source of [...(options.channels || []), ...(options.playlists || [])].slice(0, 50)) { + state.current = `browse ${source}`; await checkpoint(job, state); + try { await requireDailyBudget(); await checkpoint(job, state); const cards = await browseUploads(source, { limit: Math.max(1, Math.min(200, Number(options.resultsPerQuery) || 50)), pageDelayMs: options.delayMs, jitterMs: options.jitterMs }); await checkpoint(job, state); await saveCards(state.current, cards); log(state, `Browsed ${cards.length} videos from ${source}.`); } + catch (error) { if (error.name === 'DeepSearchStopped') throw error; state.errors.push({ query: state.current, error: String(error.message).slice(0, 200) }); } + await checkpoint(job, state); + } + state.current = ''; log(state, 'Deep search complete.'); await checkpoint(job, state, 'complete'); + } catch (error) { + if (error.name !== 'DeepSearchStopped') { + state.errors.push({ query: state.current, error: String(error.message).slice(0, 200) }); + await db.execute({ sql: "UPDATE deep_search_jobs SET state=?,status='failed',error=?,updated_at=? WHERE id=? AND status='running'", args: [JSON.stringify(state), String(error.message).slice(0, 300), now(), job.id] }); + } + } + } + async function runNext() { + if (worker) return worker; + worker = (async () => { + // Recover jobs interrupted by process restart; the current query is safe to repeat. + await db.execute({ sql: "UPDATE deep_search_jobs SET status='queued',updated_at=? WHERE status='running'", args: [now()] }); + const today = new Date(now()).toISOString().slice(0, 10); + const paused = (await db.execute("SELECT id,state FROM deep_search_jobs WHERE status='paused'")).rows; + for (const row of paused) if (parse(row.state).dailyPausedDay && parse(row.state).dailyPausedDay < today) + await db.execute({ sql: "UPDATE deep_search_jobs SET status='queued',updated_at=? WHERE id=? AND status='paused'", args: [now(), row.id] }); + const job = (await db.execute({ sql: "SELECT * FROM deep_search_jobs WHERE status='queued' ORDER BY created_at LIMIT 1" })).rows[0]; + if (job) { + await db.execute({ sql: "UPDATE deep_search_jobs SET status='running',updated_at=? WHERE id=? AND status='queued'", args: [now(), job.id] }); + const current = await getJob(job.id); if (current?.status === 'running') await run(current); + } + return !!job; + })().finally(() => { worker = null; }); + return worker; + } + const kick = () => runNext().catch(error => console.error('[deep search]', error.message)); + function start() { if (!timer) { timer = setInterval(kick, 15000); timer.unref?.(); } kick(); } + if (autoStart) start(); + async function estimate(body) { + const plan = expandQueryPlan(body); + const browseSources = [...(Array.isArray(body?.channels) ? body.channels : []), ...(Array.isArray(body?.playlists) ? body.playlists : [])].map(clean).filter(Boolean).slice(0, 50).length; + const operations = plan.length + browseSources + (Number(body?.relatedDepth) > 0 ? Math.min(plan.length * (Number(body?.relatedTop) || 3) * Math.min(2, Number(body.relatedDepth)), 100) : 0); + const today = new Date(now()).toISOString().slice(0, 10); + const used = Number((await db.execute({ sql: 'SELECT queries FROM deep_search_daily WHERE day=?', args: [today] })).rows[0]?.queries || 0); + return { plan, queries: plan.length, browseSources, estimatedOperations: operations, + estimatedMinutes: Math.ceil(operations * Math.max(1, Number(body?.delaySeconds) || 10) / 60), + indexSize: await indexSize(), dailyUsed: used, dailyRemaining: Math.max(0, (Number(body?.dailyCap) || 1000) - used) }; + } + function register(app, adminAuth) { + app.post('/api/admin/deep-search/estimate', adminAuth, async c => { let body = {}; try { body = await c.req.json(); } catch {} return c.json({ ok: true, ...(await estimate(body)) }); }); + app.get('/api/admin/deep-search/jobs', adminAuth, async c => c.json({ ok: true, indexSize: await indexSize(), jobs: (await db.execute('SELECT * FROM deep_search_jobs ORDER BY created_at DESC LIMIT 30')).rows.map(row => publicJob(row)) })); + app.post('/api/admin/deep-search/start', adminAuth, async c => { + let body; try { body = await c.req.json(); } catch { return c.json({ ok: false, error: 'Invalid JSON.' }, 400); } + const plan = expandQueryPlan(body); if (!plan.length) return c.json({ ok: false, error: 'Add at least one seed term.' }, 400); + const active = (await db.execute("SELECT id FROM deep_search_jobs WHERE status IN ('queued','running','paused') LIMIT 1")).rows[0]; + if (active) return c.json({ ok: false, error: 'A deep-search job is already active.' }, 409); + const options = { resultsPerQuery: Math.max(1, Math.min(200, Number(body.resultsPerQuery) || 50)), relatedDepth: Math.max(0, Math.min(2, Number(body.relatedDepth) || 0)), relatedTop: Math.max(1, Math.min(10, Number(body.relatedTop) || 3)), delayMs: Math.max(1000, Math.min(300000, (Number(body.delaySeconds) || 10) * 1000)), jitterMs: Math.max(0, Math.min(300000, (Number(body.jitterSeconds) || 3) * 1000)), dailyCap: Math.max(1, Math.min(10000, Number(body.dailyCap) || 1000)), channels: (Array.isArray(body.channels) ? body.channels : []).map(clean).filter(Boolean).slice(0, 25), playlists: (Array.isArray(body.playlists) ? body.playlists : []).map(clean).filter(Boolean).slice(0, 25) }; + const id = randomUUID(), time = now(); + await db.execute({ sql: "INSERT INTO deep_search_jobs(id,status,plan,options,state,created_at,updated_at) VALUES(?,'queued',?,?,?,?,?)", args: [id, JSON.stringify(plan), JSON.stringify(options), JSON.stringify({ queriesDone: 0, newVideos: 0, errors: [], log: [] }), time, time] }); + if (autoStart) kick(); return c.json({ ok: true, id }, 202); + }); + for (const action of ['pause', 'resume', 'cancel']) app.post(`/api/admin/deep-search/:id/${action}`, adminAuth, async c => { + const id = c.req.param('id'); + const allowed = action === 'pause' ? ['queued', 'running'] : action === 'resume' ? ['paused'] : ['queued', 'running', 'paused']; + const target = action === 'pause' ? 'paused' : action === 'resume' ? 'queued' : 'cancelled'; + const marks = allowed.map(() => '?').join(','); + const r = await db.execute({ sql: `UPDATE deep_search_jobs SET status=?,updated_at=? WHERE id=? AND status IN (${marks})`, args: [target, now(), id, ...allowed] }); + if (action === 'resume' && r.rowsAffected && autoStart) kick(); + return c.json({ ok: !!r.rowsAffected, status: r.rowsAffected ? target : undefined }, r.rowsAffected ? 200 : 409); + }); + return { estimate, runNext, kick, start }; + } + return { estimate, runNext, kick, start, register }; +} + +export function registerDeepSearchRoutes(app, options = {}) { + const service = createDeepSearchService(options); + service.register(app, options.adminAuth); + return service; +} diff --git a/server/deep-search.test.js b/server/deep-search.test.js new file mode 100644 index 0000000..b9710ef --- /dev/null +++ b/server/deep-search.test.js @@ -0,0 +1,122 @@ +import { expect, test, beforeAll } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Hono } from 'hono'; + +const root = mkdtempSync(join(tmpdir(), 'ytp-deep-search-')); +process.env.DB_PATH = join(root, 'test.db'); +const { initDb, db } = await import('./db.js'); +const catalog = await import('./video-catalog.js'); +const searchCache = await import('./search-cache.js'); +const { createDeepSearchService, expandQueryPlan } = await import('./deep-search.js'); +const auth = async (_c, next) => next(); +const sample = { id: 'd33psrch001', title: 'Worship in Cebuano', channel: 'Hope Choir' }; + +beforeAll(async () => { await initDb(); }); + +function makeApp(service) { + const app = new Hono(); service.register(app, auth); return app; +} +async function request(app, path, method = 'GET', body) { + const response = await app.request(path, { method, headers: body ? { 'content-type': 'application/json' } : {}, body: body ? JSON.stringify(body) : undefined }); + return { response, json: await response.json() }; +} +const cleanDb = async () => { + await db.batch(['DELETE FROM deep_search_jobs', 'DELETE FROM deep_search_daily', 'DELETE FROM video_meta', 'DELETE FROM video_meta_sources', 'DELETE FROM search_cache']); +}; + +test.serial('query plan expands modifiers, folds accents and deduplicates', () => { + expect(expandQueryPlan({ seeds: ['Adoración', 'adoracion', ' Gospel '], modifiers: ['live', 'LIVE'], maxQueries: 30 })) + .toEqual(['Adoración', 'Adoración live', 'Gospel', 'Gospel live']); + expect(expandQueryPlan({ seeds: ['one', 'two'], modifiers: ['live'], maxQueries: 2 })).toHaveLength(2); +}); + +test.serial('estimate and background job ingest to catalog, source and search cache', async () => { + await cleanDb(); + const service = createDeepSearchService({ db, ingest: catalog.ingest, putSearch: searchCache.put, + searchDeep: async () => [sample], sleep: async () => {}, autoStart: false }); + const app = makeApp(service); + const estimate = await request(app, '/api/admin/deep-search/estimate', 'POST', { seeds: ['worship'], modifiers: ['live','lyrics'], maxQueries: 4 }); + expect(estimate.json.plan).toEqual(['worship', 'worship live', 'worship lyrics']); + const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], modifiers: [], maxQueries: 1, delaySeconds: 1 }); + expect(created.response.status).toBe(202); + await service.runNext(); + const jobs = await request(app, '/api/admin/deep-search/jobs'); + expect(jobs.json.jobs[0].status).toBe('complete'); + expect(jobs.json.jobs[0].newVideos).toBe(1); + const row = (await db.execute({ sql: 'SELECT source FROM video_meta_sources WHERE video_id=?', args: [sample.id] })).rows[0]; + expect(row.source).toBe('deep'); + expect((await searchCache.get('worship')).results[0].id).toBe(sample.id); +}); + +test.serial('pause during upstream call does not ingest; resume continues from the persisted query', async () => { + await cleanDb(); let begin, release, calls = 0; + const began = new Promise(resolve => { begin = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + const service = createDeepSearchService({ db, ingest: catalog.ingest, putSearch: searchCache.put, autoStart: false, + sleep: async () => {}, searchDeep: async () => { calls++; if (calls === 1) { begin(); await gate; } return [sample]; } }); + const app = makeApp(service); + const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 }); + const work = service.runNext(); await began; + await request(app, `/api/admin/deep-search/${created.json.id}/pause`, 'POST', {}); + release(); await work; + expect((await db.execute({ sql: 'SELECT COUNT(*) AS n FROM video_meta WHERE id=?', args: [sample.id] })).rows[0].n).toBe(0); + let jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('paused'); + await request(app, `/api/admin/deep-search/${created.json.id}/resume`, 'POST', {}); + await service.runNext(); + jobs = await request(app, '/api/admin/deep-search/jobs'); + expect(jobs.json.jobs[0].status).toBe('complete'); expect(calls).toBe(2); +}); + +test.serial('cancelled in-flight job does not write its late result', async () => { + await cleanDb(); let begin, release; + const began = new Promise(resolve => { begin = resolve; }), gate = new Promise(resolve => { release = resolve; }); + const service = createDeepSearchService({ db, autoStart: false, sleep: async () => {}, + searchDeep: async () => { begin(); await gate; return [sample]; }, ingest: catalog.ingest, putSearch: searchCache.put }); + const app = makeApp(service); + const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 }); + const work = service.runNext(); await began; + await request(app, `/api/admin/deep-search/${created.json.id}/cancel`, 'POST', {}); release(); await work; + expect((await db.execute({ sql: 'SELECT COUNT(*) AS n FROM video_meta WHERE id=?', args: [sample.id] })).rows[0].n).toBe(0); + const jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('cancelled'); +}); + +test.serial('page checkpoint survives a retry without double-counting its videos', async () => { + await cleanDb(); let calls = 0; + const service = createDeepSearchService({ db, autoStart: false, sleep: async () => {}, ingest: catalog.ingest, putSearch: searchCache.put, + searchDeep: async (_query, { onPage }) => { + calls++; + await onPage([sample]); + if (calls === 1) throw Error('connection reset after first page'); + return [sample]; + } }); + const app = makeApp(service); + await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 }); + await service.runNext(); + const row = (await db.execute({ sql: 'SELECT seen FROM video_meta WHERE id=?', args: [sample.id] })).rows[0]; + expect(Number(row.seen)).toBe(1); +}); + +test.serial('daily cap pauses and worker resumes it on the next UTC day', async () => { + await cleanDb(); let time = Date.parse('2026-10-14T23:59:00Z'); + const service = createDeepSearchService({ db, now: () => time, autoStart: false, sleep: async () => {}, + searchDeep: async query => [{ ...sample, id: query === 'worship' ? 'd33psrch001' : 'd33psrch002' }], + ingest: catalog.ingest, putSearch: searchCache.put }); + const app = makeApp(service); + const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship','praise'], maxQueries: 2, delaySeconds: 1, dailyCap: 1 }); + await service.runNext(); + let jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('paused'); + time += 120000; await service.runNext(); jobs = await request(app, '/api/admin/deep-search/jobs'); + expect(jobs.json.jobs[0].status).toBe('complete'); expect(jobs.json.jobs[0].queriesDone).toBe(2); +}); + +test.serial('only one queued/running job is accepted', async () => { + await cleanDb(); + const service = createDeepSearchService({ db, autoStart: false }); const app = makeApp(service); + const body = { seeds: ['worship'], maxQueries: 1 }; + expect((await request(app, '/api/admin/deep-search/start', 'POST', body)).response.status).toBe(202); + expect((await request(app, '/api/admin/deep-search/start', 'POST', body)).response.status).toBe(409); +}); + +test.serial('cleanup test database', () => { rmSync(root, { recursive: true, force: true }); }); diff --git a/server/innertube.js b/server/innertube.js index c6ae686..cc9dbf9 100644 --- a/server/innertube.js +++ b/server/innertube.js @@ -20,6 +20,7 @@ function cardsFromSections(sections) { const path = be.canonicalBaseUrl || (be.browseId ? `/channel/${be.browseId}` : ''); out.push({ id: v.videoId, + url: `https://www.youtube.com/watch?v=${v.videoId}`, title: v.title?.runs?.map((r) => r.text).join('') || '(untitled)', channel: owner.text || '', channelId: be.browseId || '', @@ -78,7 +79,7 @@ export async function search(q, { fetchImpl = fetch, timeoutMs = 6000 } = {}) { // are a chain, so they are fetched one after another; every page is optional — // a failure part-way returns what was gathered. `onPage(cards)` lets a caller // save progress. -export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs = 6000, maxPages = 14, onPage } = {}) { +export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs = 6000, maxPages = 14, onPage, pageDelayMs = 0, jitterMs = 0 } = {}) { const post = async (body) => { const res = await fetchImpl('https://www.youtube.com/youtubei/v1/search?prettyPrint=false', { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -92,19 +93,92 @@ export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs const all = []; const add = (cards) => { for (const c of cards) if (!seen.has(c.id)) { seen.add(c.id); all.push(c); } }; add(first.cards); - if (onPage) onPage(all.slice()); + if (onPage) await onPage(all.slice()); let next = first.next; for (let i = 0; i < maxPages && next && all.length < limit; i++) { + if (pageDelayMs > 0 || jitterMs > 0) await new Promise(resolve => setTimeout(resolve, Math.max(0, pageDelayMs + Math.random() * jitterMs))); let page; try { page = parseContinuation(await post({ continuation: next })); } catch { break; } if (!page.cards.length) break; add(page.cards); - if (onPage) onPage(all.slice()); + if (onPage) await onPage(all.slice()); next = page.next; } return all.slice(0, limit); } +function browseToken(value) { + if (!value || typeof value !== 'object') return ''; + const token = value.continuationItemRenderer?.continuationEndpoint?.continuationCommand?.token; + if (token) return token; + for (const child of Object.values(value)) { + if (Array.isArray(child)) { for (const item of child) { const found = browseToken(item); if (found) return found; } } + else { const found = browseToken(child); if (found) return found; } + } + return ''; +} + +export function parseBrowseVideos(json) { + const cards = new Map(); + function visit(value) { + if (!value || typeof value !== 'object') return; + const video = value.videoRenderer || value.gridVideoRenderer || value.richItemRenderer?.content?.videoRenderer; + const id = video?.videoId; + const title = video?.title?.runs?.map(run => run.text).join('') || video?.title?.simpleText || ''; + if (id && /^[\w-]{11}$/.test(id) && title) { + const owner = video.ownerText?.runs?.[0] || video.shortBylineText?.runs?.[0] || {}; + const browse = owner.navigationEndpoint?.browseEndpoint || {}; + cards.set(id, { id, url: `https://www.youtube.com/watch?v=${id}`, title, channel: owner.text || '', + channelId: browse.browseId || '', channelUrl: browse.canonicalBaseUrl ? `https://www.youtube.com${browse.canonicalBaseUrl}` : '', + duration: lengthToSeconds(video.lengthText?.simpleText), thumbnail: `https://i.ytimg.com/vi/${id}/mqdefault.jpg` }); + return; + } + for (const child of Object.values(value)) if (Array.isArray(child)) child.forEach(visit); else visit(child); + } + visit(json); + return [...cards.values()]; +} + +// YouTube browse continuation for channel uploads and playlists. Channel IDs +// are translated to their uploads playlist (UU…), while playlist URLs use VL. +export async function browseUploads(source, { limit = 200, fetchImpl = fetch, timeoutMs = 8000, maxPages = 12, pageDelayMs = 0, jitterMs = 0 } = {}) { + let id = String(source || '').trim(); + let handle = ''; + try { + const url = new URL(id); + id = url.searchParams.get('list') || url.pathname.match(/\/channel\/([^/]+)/)?.[1] || id; + handle = url.pathname.match(/\/@([^/]+)/)?.[1] || ''; + } catch { /* raw channel or playlist id */ } + if (handle) { + const response = await fetchImpl(`https://www.youtube.com/@${encodeURIComponent(handle)}`, { signal: AbortSignal.timeout(timeoutMs) }); + if (!response.ok) throw new Error(`channel lookup: HTTP ${response.status}`); + const html = await response.text(); + id = html.match(/"browseId"\s*:\s*"(UC[\w-]{22})"/)?.[1] || ''; + } + let browseId; + if (/^UC[\w-]{22}$/.test(id)) browseId = `UU${id.slice(2)}`; + else if (/^(?:PL|UU|LL|FL|OL)[\w-]{10,}$/.test(id)) browseId = id.startsWith('UU') || id.startsWith('LL') || id.startsWith('FL') || id.startsWith('OL') ? id : `VL${id}`; + else throw new Error('Expected a YouTube channel ID or playlist ID/URL.'); + const post = async body => { + const response = await fetchImpl('https://www.youtube.com/youtubei/v1/browse?prettyPrint=false', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ context: { client: CLIENT }, ...body }), signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) throw new Error(`innertube browse: HTTP ${response.status}`); + return response.json(); + }; + let page = await post({ browseId }); + const seen = new Set(), cards = []; + for (let i = 0; i < maxPages && cards.length < limit; i++) { + for (const card of parseBrowseVideos(page)) if (!seen.has(card.id)) { seen.add(card.id); cards.push(card); } + const token = browseToken(page); + if (!token || cards.length >= limit) break; + if (pageDelayMs > 0 || jitterMs > 0) await new Promise(resolve => setTimeout(resolve, Math.max(0, pageDelayMs + Math.random() * jitterMs))); + page = await post({ continuation: token }); + } + return cards.slice(0, limit); +} + // Watch-next recommendations use compact renderers or the newer lockup model. // Traverse only the recommendation contents, never player configuration/ads. export function parseRelated(json) { diff --git a/server/innertube.test.js b/server/innertube.test.js index da1011c..22ac50b 100644 --- a/server/innertube.test.js +++ b/server/innertube.test.js @@ -7,7 +7,7 @@ test('parses video renderers into slim cards', () => { const r = parseSearch(fixture); expect(r.length).toBe(4); expect(r[0]).toEqual({ - id: 'nQWFzMvCfLE', title: 'What A Beautiful Name - Hillsong Worship', channel: 'Hillsong Worship', + id: 'nQWFzMvCfLE', url: 'https://www.youtube.com/watch?v=nQWFzMvCfLE', title: 'What A Beautiful Name - Hillsong Worship', channel: 'Hillsong Worship', channelId: 'UC4q12NoPNySbVqwpw4iO5Vg', channelUrl: 'https://www.youtube.com/channel/UC4q12NoPNySbVqwpw4iO5Vg', duration: 343, thumbnail: 'https://i.ytimg.com/vi/nQWFzMvCfLE/mqdefault.jpg', }); @@ -60,3 +60,18 @@ test('related HTTP failures and unsupported shapes throw for search fallback', a expect(() => parseRelated({ error: {} })).toThrow('response shape'); await expect(related('aaaaaaaaaaa', { fetchImpl: async () => ({ ok: false, status: 429 }) })).rejects.toThrow('429'); }); + +test('channel and playlist browse parses uploads and resolves handle URLs', async () => { + const { browseUploads, parseBrowseVideos } = await import('./innertube.js'); + const renderer = { videoRenderer: { videoId: 'abcdefghijk', title: { simpleText: 'Upload' }, ownerText: { runs: [{ text: 'Choir', navigationEndpoint: { browseEndpoint: { browseId: 'UC1234567890123456789012', canonicalBaseUrl: '/@choir' } } }] }, lengthText: { simpleText: '2:10' } } }; + expect(parseBrowseVideos({ contents: [{ gridVideoRenderer: { videoId: 'abcdefghijk', title: { simpleText: 'Upload' } } }] })[0].title).toBe('Upload'); + const calls = []; + const fetchImpl = async (url, init = {}) => { + calls.push([String(url), init.body && JSON.parse(init.body)]); + if (String(url).includes('/@choir')) return new Response('"browseId":"UC1234567890123456789012"'); + return Response.json({ contents: [{ gridVideoRenderer: renderer.videoRenderer }] }); + }; + const result = await browseUploads('https://www.youtube.com/@choir', { fetchImpl, limit: 5 }); + expect(result[0].id).toBe('abcdefghijk'); + expect(calls[1][1].browseId).toBe('UU1234567890123456789012'); +}); diff --git a/server/media-cache.js b/server/media-cache.js index 0496912..14bc7cb 100644 --- a/server/media-cache.js +++ b/server/media-cache.js @@ -158,7 +158,7 @@ function metaFromInfo(info, duration) { // getInfo async (videoId) => yt-dlp -J info — duration, live flags, meta // ---------------------------------------------------------------------------- export function createMediaCache({ - dir, db, download, getInfo, + dir, db, download, getInfo, getSourceUrl = id => `https://www.youtube.com/watch?v=${id}`, ffmpeg = 'ffmpeg', ffprobe = 'ffprobe', nice = 'nice', maxBytes = 10 * 1024 ** 3, minFreeBytes = 5 * 1024 ** 3, @@ -314,7 +314,9 @@ export function createMediaCache({ await job.persisted; try { await setStatus(job, 'downloading'); - const info = await getInfo(id); + const sourceUrl = await getSourceUrl(id); + if (!sourceUrl || !/^https:\/\/www\.youtube\.com\/watch\?v=[\w-]{11}$/.test(sourceUrl)) throw new MediaSkip('source URL is unavailable'); + const info = await getInfo(id, sourceUrl); const live = info.is_live || ['is_live', 'post_live', 'is_upcoming'].includes(info.live_status); if (live) throw new MediaSkip('live streams are not cached'); const expected = typeof info.duration === 'number' ? info.duration : 0; @@ -324,7 +326,7 @@ export function createMediaCache({ } await makeRoom(id, Math.max(expected, 60) * EST_BYTES_PER_SEC); - const raw = await download(id, `${base}.dl.mp4`); + const raw = await download(id, `${base}.dl.mp4`, sourceUrl); await setStatus(job, 'validating'); // Normalise: first video + first audio, moov atom up front so the copy diff --git a/server/media-cache.test.js b/server/media-cache.test.js index c44ffc4..a422c06 100644 --- a/server/media-cache.test.js +++ b/server/media-cache.test.js @@ -2,7 +2,7 @@ // Uses real ffmpeg fixtures and the real db.js against a throwaway DB file. import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { spawnSync } from 'node:child_process'; -import { mkdtempSync, rmSync, copyFileSync, writeFileSync, readFileSync, existsSync, readdirSync } from 'node:fs'; +import { mkdtempSync, rmSync, copyFileSync, writeFileSync, readFileSync, existsSync, readdirSync, unlinkSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -93,6 +93,7 @@ let dirN = 0; function makeCache({ fixture = 'good.mp4', duration = 10, ...opts } = {}) { const dir = join(root, 'media' + dirN++); const calls = []; + const sources = []; const db = { getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia, listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia, @@ -102,13 +103,13 @@ function makeCache({ fixture = 'good.mp4', duration = 10, ...opts } = {}) { const cache = createMediaCache({ dir, db, log: silentLog, getInfo: async (id) => ({ id, title: 'T ' + id, channel: 'C', duration }), - download: async (id, out) => { calls.push(id); await sleep(30); copyFileSync(fx(state.fixture), out); return out; }, + download: async (id, out, sourceUrl) => { calls.push(id); sources.push({ id, sourceUrl }); await sleep(30); copyFileSync(fx(state.fixture), out); return out; }, transcode: { enabled: false }, freeBytes: () => 100 * 1024 ** 3, backfillDelayMs: -1, ...opts, }); - return { cache, dir, calls, state }; + return { cache, dir, calls, sources, state }; } async function clearDb() { for (const r of await dbmod.listMedia()) await dbmod.deleteMedia(r.video_id); @@ -233,6 +234,19 @@ describe('media cache jobs', () => { expect(calls.length).toBe(2); }); + test('missing cached files are re-fetched from the stored source URL', async () => { + await clearDb(); + const id = 'srcAAAAAA01'; + const sourceUrl = `https://www.youtube.com/watch?v=${id}`; + const { cache, dir, calls, sources } = makeCache({ getSourceUrl: async () => sourceUrl }); + await cache.init(); + const first = await cache.ensureCached(id, { priority: HIGH }); + unlinkSync(join(dir, `${id}.${first.gen}.mp4`)); + await cache.ensureCached(id, { priority: HIGH }); + expect(calls).toHaveLength(2); + expect(sources.every(call => call.id === id && call.sourceUrl === sourceUrl)).toBe(true); + }); + test('auto jobs respect the auto length limit; explicit saves do not', async () => { await clearDb(); const { cache } = makeCache({ duration: 7200, autoMaxSeconds: 3600 }); diff --git a/server/notes.js b/server/notes.js index 88e1fbb..bf68373 100644 --- a/server/notes.js +++ b/server/notes.js @@ -314,6 +314,7 @@ export function verifyAdminCookie(secret, value, nowSec = Math.floor(Date.now() export function registerNoteRoutes(app, deps) { const { db, getProfile, profileNameRe, runYtdlp, adminPassword, backupDir, adminHtmlPath, workerToken, + catalog = null, media = null, } = deps; const ADMIN_COOKIE = 'ytp_admin'; const ADMIN_TTL = 30 * 24 * 3600; @@ -695,6 +696,7 @@ export function registerNoteRoutes(app, deps) { const n = await db.getNotes(r.video_id); out.push({ id: r.video_id, cacheStatus: r.status, title: meta.title || '', channel: meta.channel || meta.uploader || '', + sourceUrl: catalog ? await catalog.getSourceUrl(r.video_id) : null, duration: Number(r.duration) || 0, lastAccess: Number(r.last_access) || 0, lyricsRev: n.lyrics ? n.lyrics.rev : 0, lyricsLines: n.lyrics ? n.lyrics.data.lines.length : 0, }); @@ -703,6 +705,12 @@ export function registerNoteRoutes(app, deps) { return c.json({ ok: true, media: out }); }); + app.post('/api/admin/media/:id/redownload', requireAdminOrToken, async c => { + if (!media) return c.json({ ok: false, error: 'media cache unavailable' }, 503); + try { return c.json({ ok: true, ...(await media.redownload(c.req.param('id'))) }, 202); } + catch (err) { return c.json({ ok: false, error: err.message }, 400); } + }); + app.get('/api/admin/notes/recent', requireAdmin, async (c) => c.json({ ok: true, revs: await db.recentNoteRevs(150) })); app.post('/api/admin/notes/:id/:kind/restore', requireAdmin, async (c) => { diff --git a/server/notes.test.js b/server/notes.test.js index cd2a66f..65a9d7a 100644 --- a/server/notes.test.js +++ b/server/notes.test.js @@ -57,6 +57,8 @@ beforeAll(async () => { backupDir: join(root, 'backups'), adminHtmlPath: htmlPath, workerToken: 'w'.repeat(32), + catalog: { getSourceUrl: async id => /^[\w-]{11}$/.test(id) ? `https://www.youtube.com/watch?v=${id}` : null }, + media: { redownload: async id => ({ status: `queued:${id}` }) }, }); }); @@ -278,5 +280,7 @@ test('admin song list includes metadata-only phone songs for lyrics lookup', asy const cookie = await adminCookie(); const res = await app.request('/api/admin/media', { headers: { Cookie: cookie } }); const song = (await res.json()).media.find(x => x.id === id); - expect(song).toMatchObject({ title: 'Phone song', channel: 'Phone artist', duration: 240, cacheStatus: 'failed' }); + expect(song).toMatchObject({ title: 'Phone song', channel: 'Phone artist', duration: 240, cacheStatus: 'failed', sourceUrl: 'https://www.youtube.com/watch?v=phoneSong01' }); + expect((await app.request(`/api/admin/media/${id}/redownload`, { method: 'POST', headers: { Cookie: cookie } })).status).toBe(202); + expect((await app.request('/api/admin/media/phoneSong01/redownload', { method: 'POST' })).status).toBe(401); }); diff --git a/server/package.json b/server/package.json index a4e37cf..c144a43 100644 --- a/server/package.json +++ b/server/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "bun server.js", "dev": "bun --hot server.js", - "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js" + "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js" }, "dependencies": { "@hono/node-server": "^1.14.0", diff --git a/server/search-cache.js b/server/search-cache.js index 423ec32..934ad35 100644 --- a/server/search-cache.js +++ b/server/search-cache.js @@ -2,7 +2,7 @@ // repeat query — from any device, or after a restart — never waits on YouTube. // Capped by row count (default 250,000 queries) AND bytes (default 2 GiB), LRU. import { db } from './db.js'; -import { ingest, withLocalThumbnails, trimCatalog } from './video-catalog.js'; +import { ingest, withLocalThumbnails, trimCatalog, foldSearchText } from './video-catalog.js'; export const MAX_ROWS = Number(process.env.SEARCH_CACHE_MAX_ROWS) || 250_000; export const MAX_BYTES = Number(process.env.SEARCH_CACHE_MAX_BYTES) || 2 * 1024 ** 3; @@ -70,15 +70,28 @@ export async function rememberVideos(cards) { // Cards whose title/channel contain EVERY word of the query, most-seen first. export async function searchVideos(q, limit = 60) { - const words = String(q).toLowerCase().split(/\s+/).filter(Boolean).slice(0, 8); + const folded = foldSearchText(q).trim(); + const words = folded.split(/\s+/).filter(Boolean).slice(0, 8); if (!words.length) return []; const like = (w) => '%' + w.replace(/[\\%_]/g, '\\$&') + '%'; const r = await db.execute({ - sql: `SELECT card FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')} + sql: `SELECT card,seen FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')} ORDER BY seen DESC, updated_at DESC LIMIT ?`, - args: [...words.map(like), limit], + args: [...words.map(like), Math.max(limit * 10, 200)], }); - return withLocalThumbnails(r.rows.map((x) => { try { return JSON.parse(x.card); } catch { return null; } }).filter(Boolean)); + const titleHasAll = title => words.every(word => foldSearchText(title).includes(word)); + const phrase = folded; + const ranked = r.rows.map(row => { + try { + const card = JSON.parse(row.card); + const title = foldSearchText(card.title); + const channel = foldSearchText(card.channel); + const score = (title === phrase || title.includes(phrase) ? 4 : 0) + + (titleHasAll(card.title) ? 2 : 0) + (words.some(word => channel.includes(word)) ? 1 : 0); + return { card: { ...card, url: card.url || `https://www.youtube.com/watch?v=${card.id}` }, score, seen: Number(row.seen) || 0 }; + } catch { return null; } + }).filter(Boolean).sort((a, b) => b.score - a.score || b.seen - a.seen).slice(0, limit).map(x => x.card); + return withLocalThumbnails(ranked); } export async function trimVideos(max = MAX_VIDEOS) { diff --git a/server/search-cache.test.js b/server/search-cache.test.js index 47a4033..3322f98 100644 --- a/server/search-cache.test.js +++ b/server/search-cache.test.js @@ -1,8 +1,13 @@ -import { test, expect } from 'bun:test'; -import * as sc from './search-cache.js'; -import { initDb } from './db.js'; - +import { test, expect, afterAll } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +const root = mkdtempSync(join(tmpdir(), 'ytp-search-cache-')); +process.env.DB_PATH = join(root, 'test.db'); +const sc = await import('./search-cache.js'); +const { initDb } = await import('./db.js'); await initDb(); +afterAll(() => rmSync(root, { recursive: true, force: true })); const cards = (n) => Array.from({ length: n }, (_, i) => ({ id: 'v' + i, title: 't' + i })); test('put/get round-trips, case-insensitive, capped at 200', async () => { @@ -21,11 +26,11 @@ test('trim evicts least-recently-used rows over the row cap', async () => { test('known videos are remembered and found by every word', async () => { await sc.rememberVideos([ - { id: 'a1', title: 'Way Maker - Sinach', channel: 'Worship' }, - { id: 'b2', title: 'Way Maker (live)', channel: 'Other' }, - { id: 'c3', title: 'Goodness of God', channel: 'Worship' }, + { id: 'aaa11111111', title: 'Way Maker - Sinach', channel: 'Worship' }, + { id: 'bbb22222222', title: 'Way Maker (live)', channel: 'Other' }, + { id: 'ccc33333333', title: 'Goodness of God', channel: 'Worship' }, ]); - expect((await sc.searchVideos('way maker')).map((c) => c.id).sort()).toEqual(['a1', 'b2']); - expect((await sc.searchVideos('maker worship')).map((c) => c.id)).toEqual(['a1']); + expect((await sc.searchVideos('way maker')).map((c) => c.id).sort()).toEqual(['aaa11111111', 'bbb22222222']); + expect((await sc.searchVideos('maker worship')).map((c) => c.id)).toEqual(['aaa11111111']); expect(await sc.searchVideos('100%')).toEqual([]); }); diff --git a/server/server.js b/server/server.js index 771d7dc..376ec94 100644 --- a/server/server.js +++ b/server/server.js @@ -68,6 +68,7 @@ import { registerCatalogRoutes } from './recommendations.js'; import QRCode from 'qrcode'; import { createYtdlpPool } from './ytdlp-pool.js'; import { createWarmQueue } from './warm-queue.js'; +import { registerDeepSearchRoutes } from './deep-search.js'; import { dirname, join as pathJoin } from 'node:path'; // A media proxy must not die because one client's stream hit an edge case @@ -268,6 +269,7 @@ function slimEntry(j) { return { id, title: pick(j, 'title') || '(untitled)', + url: /^[\w-]{11}$/.test(id) ? `https://www.youtube.com/watch?v=${id}` : null, channel: pickChannel(j), channelId: pickChannelId(j), channelUrl: pickChannelUrl(j), @@ -418,6 +420,8 @@ const searchLibrary = async (q) => { // This server's own library first — it still answers when YouTube is unreachable. try { return (await notesDb.listUploads({ q, limit: 20, listedOnly: true })).map(uploads.card); } catch { return []; } }; +const withYoutubeUrl = card => card && /^[\w-]{11}$/.test(card.id || '') + ? { ...card, url: `https://www.youtube.com/watch?v=${card.id}` } : card; // GET /api/search/local?q= — videos this server already knows (from every // earlier search) plus its own uploads; answers in milliseconds so the app can @@ -426,7 +430,7 @@ app.get('/api/search/local', async (c) => { const q = (c.req.query('q') || '').trim(); if (!q) return c.json({ ok: false, error: 'empty query' }, 400); const [mine, known] = await Promise.all([searchLibrary(q), searchCacheDb.searchVideos(q).catch(() => [])]); - return c.json({ ok: true, results: [...mine, ...known], local: true }); + return c.json({ ok: true, results: [...mine, ...known].map(withYoutubeUrl), local: true }); }); // GET /api/search?q=[&refresh=1] @@ -439,7 +443,7 @@ app.get('/api/search', async (c) => { if (!q) return c.json({ ok: false, error: 'empty query' }, 400); const key = q.toLowerCase(); const mine = await searchLibrary(q); - const answer = (yt, extra = {}) => c.json({ ok: true, results: [...mine, ...yt], ...extra }); + const answer = (yt, extra = {}) => c.json({ ok: true, results: [...mine, ...yt].map(withYoutubeUrl), ...extra }); if (c.req.query('refresh') !== '1') { let hit = searchCache.get(key); if (!hit) { @@ -1101,7 +1105,7 @@ const SAVE_DEFAULT_FORMAT = // again) share ONE yt-dlp run instead of spawning another each time. const inflightDownloads = new Map(); // key -> { promise, waiters, tmpBase } -async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) { +async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal, sourceUrl = null) { await assertNotLive(videoId); const key = videoId + '|' + formatArgs.join(' '); let entry = inflightDownloads.get(key); @@ -1110,10 +1114,11 @@ async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) { const tmp = `${tmpdir()}/${tmpBase}.mp4`; const ctl = new AbortController(); entry = { waiters: 0, tmpBase, ctl }; - entry.promise = withSaveSlot(() => { + entry.promise = withSaveSlot(async () => { if (ctl.signal.aborted) throw new Error('save cancelled'); + const url = sourceUrl || await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`; return runYtdlpResilient([ - `https://www.youtube.com/watch?v=${videoId}`, + url, '--no-warnings', '--no-playlist', ...formatArgs, '--limit-rate', DOWNLOAD_RATE, @@ -1183,14 +1188,16 @@ async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal, cachedSrc try { // 1) Grab the full source (video+audio merged) so ffmpeg has both streams // — unless the server cache already holds it. - if (!cachedSrc) await withSaveSlot(() => runYtdlpResilient([ - `https://www.youtube.com/watch?v=${videoId}`, - '--no-warnings', '--no-playlist', - '-f', SAVE_MUX_FORMAT, - '--merge-output-format', 'mp4', - '--limit-rate', DOWNLOAD_RATE, - '-o', srcTmp, - ], { signal })); + if (!cachedSrc) await withSaveSlot(async () => { + const url = await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`; + return runYtdlpResilient([ + url, '--no-warnings', '--no-playlist', + '-f', SAVE_MUX_FORMAT, + '--merge-output-format', 'mp4', + '--limit-rate', DOWNLOAD_RATE, + '-o', srcTmp, + ], { signal }); + }); // 2) Trim + concat the keep segments into the final custom video. await runFfmpeg([ '-y', '-hide_banner', '-loglevel', 'error', @@ -1240,6 +1247,7 @@ if (!MEDIA_VOLUME_MARKER || existsSync(MEDIA_VOLUME_MARKER)) mkdirSync(MEDIA_DIR const media = createMediaCache({ dir: MEDIA_DIR, volumeMarker: MEDIA_VOLUME_MARKER, + getSourceUrl: videoCatalog.getSourceUrl, db: { getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats, // Retention (docs/p2p-architecture.md flow 9): cold copies go before popular ones. @@ -1248,9 +1256,9 @@ const media = createMediaCache({ }), }, getInfo: async (videoId) => (await resolveStreams(videoId)).info, - download: async (videoId, out) => { + download: async (videoId, out, sourceUrl) => { await withSaveSlot(() => runYtdlpResilient([ - `https://www.youtube.com/watch?v=${videoId}`, + sourceUrl || `https://www.youtube.com/watch?v=${videoId}`, '--no-warnings', '--no-playlist', '-f', SAVE_MUX_FORMAT, '--merge-output-format', 'mp4', @@ -2291,6 +2299,8 @@ app.get('/api/remote/qr/:code', async (c) => { // ============================================================================ const notes = registerNoteRoutes(app, { db: notesDb, + catalog: videoCatalog, + media, getProfile, profileNameRe: PROFILE_NAME_RE, runYtdlp: runYtdlpResilient, @@ -2304,6 +2314,9 @@ const notes = registerNoteRoutes(app, { { label: 'Database', path: dirname(process.env.DB_PATH || './data/ytplayer.db') }, ] }, }); +const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, autoStart: false, + searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads, + ingest: collectVideoMetadata, putSearch: searchCacheDb.put }); // ============================================================================ // Uploads — the server's own searchable media library (see uploads.js) @@ -2515,6 +2528,7 @@ app.get('/*', indexHtml); // ============================================================================ async function main() { await initDb(); + deepSearch.start(); startThumbnails(); await p2pDb.initP2pSchema(); console.log(`[ytplayer] DB ready`); diff --git a/server/video-catalog.js b/server/video-catalog.js index 081af55..5424d3e 100644 --- a/server/video-catalog.js +++ b/server/video-catalog.js @@ -3,11 +3,18 @@ import { db } from './db.js'; import { mergeMissing } from './media-meta-core.js'; const VIDEO_ID = /^[\w-]{11}$/; -const SOURCES = new Set(['search', 'search-cache', 'client-search', 'channel', 'streams', 'playlist', 'profile', 'sync', 'related', 'backfill', 'collector', 'device-play']); +const SOURCES = new Set(['search', 'search-cache', 'client-search', 'channel', 'streams', 'playlist', 'profile', 'sync', 'related', 'backfill', 'collector', 'deep', 'device-play']); const text = (v, n = 300) => typeof v === 'string' ? v.trim().slice(0, n) : ''; const positive = v => Number.isFinite(Number(v)) && Number(v) > 0 ? Number(v) : 0; const canonicalThumb = id => `https://i.ytimg.com/vi/${id}/hqdefault.jpg`; +const canonicalUrl = id => `https://www.youtube.com/watch?v=${id}`; +export const foldSearchText = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase(); export const channelKey = c => String(c.channelId || c.channel || '').toLowerCase().replace(/\s*-\s*topic$|vevo$|\s+official$/g, '').trim(); +export async function getSourceUrl(id) { + if (!VIDEO_ID.test(String(id || ''))) return null; + const row = (await db.execute({ sql: 'SELECT url FROM video_meta WHERE id=?', args: [id] })).rows[0]; + return row?.url || `https://www.youtube.com/watch?v=${id}`; +} function safeThumbnail(url) { try { const u = new URL(url); return u.protocol === 'https:' && /^(?:i|i\d)\.ytimg\.com$/.test(u.hostname) && !u.port && !u.username && !u.password; } catch { return false; } @@ -23,14 +30,16 @@ export function normalizeCard(v) { // Never fetch arbitrary client URLs (including redirects or private hosts). if (safeThumbnail(u.href)) thumbnail = u.href; } catch { /* canonical YouTube art */ } - const card = { id: v.id, title, channel: text(v.channel), channelId: text(v.channelId, 64), + const card = { id: v.id, title, url: canonicalUrl(v.id), channel: text(v.channel), channelId: text(v.channelId ?? v.channel_id, 64), duration: positive(v.duration), thumbnail }; try { - const u = new URL(v.channelUrl); + const u = new URL(v.channelUrl || v.channel_url); if (u.protocol === 'https:' && /^(www\.)?youtube\.com$/.test(u.hostname)) card.channelUrl = u.href; } catch { /* optional */ } for (const key of ['tags', 'categories']) if (Array.isArray(v[key])) card[key] = v[key].slice(0, 30).map(x => text(x, 80)).filter(Boolean); if (v.description) card.description = text(v.description, 1200); + if (v.published || v.publishedAt || v.upload_date) card.published = text(v.published || v.publishedAt || v.upload_date, 40); + if (v.lang || v.language) card.lang = text(v.lang || v.language, 24); if (positive(v.viewCount ?? v.view_count)) card.viewCount = positive(v.viewCount ?? v.view_count); return card; } @@ -67,9 +76,12 @@ async function ingestBatch(cards, source, fillMissing) { await db.batch(batch.flatMap(c => { // Sparse playlist cards must not erase richer extractor metadata. const merged = fillMissing ? mergeMissing(old.get(c.id) || {}, c) : { ...(old.get(c.id) || {}), ...Object.fromEntries(Object.entries(c).filter(([, v]) => v !== '' && v !== 0 && (!Array.isArray(v) || v.length))) }; - return [{ sql: `INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,1,?) - ON CONFLICT(id) DO UPDATE SET card=excluded.card,hay=excluded.hay,seen=seen+1,updated_at=excluded.updated_at`, - args: [c.id, JSON.stringify(merged), `${merged.title} ${merged.channel || ''} ${(merged.tags || []).join(' ')}`.toLowerCase(), now] }, + const hay = foldSearchText([merged.title, merged.channel, merged.channelId, merged.description, + ...(merged.tags || []), ...(merged.categories || [])].filter(Boolean).join(' ')); + return [{ sql: `INSERT INTO video_meta (id,card,hay,seen,updated_at,url,channel_id,channel_url,published,lang) VALUES (?,?,?,1,?,?,?,?,?,?) + ON CONFLICT(id) DO UPDATE SET card=excluded.card,hay=excluded.hay,seen=seen+1,updated_at=excluded.updated_at, + url=excluded.url,channel_id=excluded.channel_id,channel_url=excluded.channel_url,published=excluded.published,lang=excluded.lang`, + args: [c.id, JSON.stringify(merged), hay, now, merged.url, merged.channelId || null, merged.channelUrl || null, merged.published || null, merged.lang || null] }, { sql: `INSERT INTO video_meta_sources (video_id,source,last_seen) VALUES (?,?,?) ON CONFLICT(video_id,source) DO UPDATE SET discoveries=discoveries+1,last_seen=excluded.last_seen`, args: [c.id, source, now] }, { sql: `INSERT INTO video_thumbnails (video_id,url) VALUES (?,?) ON CONFLICT(video_id) DO UPDATE SET retry_at=0 diff --git a/server/video-catalog.test.js b/server/video-catalog.test.js new file mode 100644 index 0000000..eb0c205 --- /dev/null +++ b/server/video-catalog.test.js @@ -0,0 +1,53 @@ +import { test, expect } from 'bun:test'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const root = mkdtempSync(join(tmpdir(), 'ytp-video-catalog-')); +process.env.DB_PATH = join(root, 'test.db'); +const db = await import('./db.js'); +const catalog = await import('./video-catalog.js'); +await db.initDb(); + +test('catalog migration backfills canonical URLs and rich metadata, excluding upload ids', async () => { + await db.db.execute('DELETE FROM catalog_schema_migrations WHERE version=1'); + await db.db.execute({ sql: 'INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,?,?)', + args: ['abcDEF12345', JSON.stringify({ id: 'abcDEF12345', title: 'Café Worship', channel: 'Choir', channelId: 'UC123', channelUrl: 'https://www.youtube.com/@choir', tags: ['live'], publishedAt: '2024-01-01', language: 'en' }), 'cafe worship choir', 1, 1] }); + await db.db.execute({ sql: 'INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,?,?)', + args: ['upl_abcdef12', JSON.stringify({ id: 'upl_abcdef12', title: 'Upload' }), 'upload', 1, 1] }); + await db.migrateVideoMeta({ pageSize: 1 }); + await db.migrateVideoMeta({ pageSize: 1 }); + const youtube = (await db.db.execute("SELECT * FROM video_meta WHERE id='abcDEF12345'")).rows[0]; + const upload = (await db.db.execute("SELECT * FROM video_meta WHERE id='upl_abcdef12'")).rows[0]; + expect(youtube.url).toBe('https://www.youtube.com/watch?v=abcDEF12345'); + expect(youtube.channel_id).toBe('UC123'); + expect(youtube.channel_url).toBe('https://www.youtube.com/@choir'); + expect(youtube.published).toBe('2024-01-01'); + expect(youtube.lang).toBe('en'); + expect(youtube.hay).toContain('cafe'); + expect(JSON.parse(youtube.card).url).toBe(youtube.url); + expect(upload.url).toBeNull(); +}); + +test('ingest stores canonical source URL and channel metadata on the card and row', async () => { + await catalog.ingest([{ id: 'xyzABC12345', title: 'Música', channel: 'Choir', channel_id: 'UC456', channel_url: 'https://www.youtube.com/@choir', tags: ['Praise'], description: 'Sunday service' }], 'deep'); + const row = (await db.db.execute("SELECT * FROM video_meta WHERE id='xyzABC12345'")).rows[0]; + const card = JSON.parse(row.card); + expect(row.url).toBe('https://www.youtube.com/watch?v=xyzABC12345'); + expect(row.channel_id).toBe('UC456'); + expect(card.url).toBe(row.url); + expect(row.hay).toContain('musica'); + expect(row.hay).toContain('praise'); +}); + +test('accent-insensitive matching ranks title phrase ahead of channel match', async () => { + await catalog.ingest([ + { id: 'phrase12345', title: 'Café Worship Live', channel: 'Other' }, + { id: 'channel1234', title: 'Sunday Set', channel: 'Cafe Worship Live' }, + { id: 'words123456', title: 'Worship in the Cafe', channel: 'Other' }, + ], 'deep'); + const search = await import('./search-cache.js'); + const results = await search.searchVideos('cafe worship live'); + expect(results[0].id).toBe('phrase12345'); + expect(results[0].url).toBe('https://www.youtube.com/watch?v=phrase12345'); +}); diff --git a/tests/admin-deep-search.spec.js b/tests/admin-deep-search.spec.js new file mode 100644 index 0000000..7da7de2 --- /dev/null +++ b/tests/admin-deep-search.spec.js @@ -0,0 +1,38 @@ +const { test, expect } = require('@playwright/test'); + +test('admin deep-search tab estimates, starts, controls and shows persisted progress', async ({ page }) => { + page.on('pageerror', error => console.error('pageerror:', error.message)); + const state = { jobs: [], events: [] }; + await page.route('**/api/**', async route => { + const req = route.request(), url = new URL(req.url()); + let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true }; + else if (url.pathname === '/api/admin/media') body = { ok: true, media: [] }; + else if (url.pathname === '/api/uploads') body = { ok: true, uploads: [] }; + else if (url.pathname === '/api/admin/deep-search/estimate') body = { ok: true, plan: ['worship', 'worship live'], queries: 2, estimatedMinutes: 1, indexSize: 1240, dailyRemaining: 999 }; + else if (url.pathname === '/api/admin/deep-search/start') { + state.events.push('start'); state.jobs = [{ id: 'deep-1', status: 'running', plan: ['worship', 'worship live'], options: {}, queriesDone: 1, queriesTotal: 2, current: 'worship live', newVideos: 24, totalIndexSize: 1264, etaMs: 12000, errors: [], log: [{ at: Date.now(), message: 'Saved 24 results' }] }]; + body = { ok: true, id: 'deep-1' }; + } else if (url.pathname === '/api/admin/deep-search/jobs') body = { ok: true, indexSize: 1264, jobs: state.jobs }; + else if (/\/api\/admin\/deep-search\/deep-1\/(pause|resume|cancel)$/.test(url.pathname)) { + const action = url.pathname.split('/').at(-1); state.events.push(action); + state.jobs[0].status = action === 'pause' ? 'paused' : action === 'resume' ? 'running' : 'cancelled'; + body = { ok: true, status: state.jobs[0].status }; + } + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.goto('/admin.html#deep-search'); + await expect(page.locator('#deepSearchForm')).toBeVisible(); + await expect(page.locator('#deepSeeds')).toHaveValue(/worship/); + await page.locator('#deepEstimate').click(); + await expect(page.locator('#deepEstimateText')).toContainText('2 unique queries'); + await page.locator('#deepStart').click(); + await expect(page.locator('.deep-job')).toContainText('1/2 queries'); + await page.locator('[data-deep-action="pause"]').click(); + await expect(page.locator('.deep-job')).toContainText('Paused'); + await page.locator('[data-deep-action="resume"]').click(); + await expect(page.locator('.deep-job')).toContainText('Running'); + await page.locator('[data-deep-action="cancel"]').click(); + await expect(page.locator('.deep-job')).toContainText('Cancelled'); + expect(state.events).toEqual(['start', 'pause', 'resume', 'cancel']); +});