From 58d0e6f1401f61e7cb4e4b4a05d5914792c1ce9a Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 01:47:35 +0800 Subject: [PATCH] Add resumable admin deep search --- frontend/admin.html | 93 ++++++++++- playwright.admin-deep-search.config.js | 3 + server/db.js | 9 ++ server/deep-search.js | 214 +++++++++++++++++++++++++ server/deep-search.test.js | 122 ++++++++++++++ server/innertube.js | 79 ++++++++- server/innertube.test.js | 17 +- server/package.json | 2 +- server/search-cache.test.js | 13 +- server/server.js | 30 ++-- server/video-catalog.js | 2 +- tests/admin-deep-search.spec.js | 38 +++++ 12 files changed, 600 insertions(+), 22 deletions(-) create mode 100644 playwright.admin-deep-search.config.js create mode 100644 server/deep-search.js create mode 100644 server/deep-search.test.js create mode 100644 tests/admin-deep-search.spec.js diff --git a/frontend/admin.html b/frontend/admin.html index 0d8cbc4..3d1d515 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -280,6 +280,14 @@ .collection-job p { margin: 4px 0; color: var(--text-2); } #metadataDetail { white-space: pre-wrap; overflow-wrap: anywhere; max-height: 400px; overflow: auto; font-size: 13px; } @media (max-width: 480px) { .analytics-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } .metric { padding: 12px; } .metric b { font-size: 22px; } } + .deep-grid { display: grid; grid-template-columns: repeat(auto-fit,minmax(min(100%,240px),1fr)); gap: 12px; } + .deep-grid label { display: grid; gap: 5px; min-width: 0; color: var(--text); font-size: 13px; } + .deep-search .muted { color: var(--text-2); } + .deep-grid .field { width: 100%; flex: initial; } + .deep-grid textarea.field { height: auto; min-height: 110px; padding: 10px 12px; resize: vertical; } + .deep-job { padding: 14px 0; border-bottom: 1px solid var(--line-2); overflow-wrap: anywhere; } + .deep-job progress { width: 100%; height: 8px; accent-color: var(--a1); } + .deep-log { max-height: 240px; overflow: auto; white-space: pre-wrap; font: 12px/1.5 var(--mono); color: var(--text-2); } @media (min-width: 761px) and (max-width: 1100px) { .top { flex-wrap: wrap; } .tabs { order: 2; width: 100%; margin-left: 0; overflow-x: auto; } @@ -547,6 +555,29 @@

Most played

+ +

API access

Tokens let scripts write shared lyrics and chapters. A token is shown once — store it in your secrets vault.

@@ -608,9 +639,10 @@ peers: '', history: '', analytics: '', + 'deep-search': '', access: '', }; - const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -630,6 +662,7 @@ if (name === 'library') loadUploads(); if (name === 'access') loadTokens(); if (name === 'analytics') loadAnalytics(); + if (name === 'deep-search') loadDeepSearch(); window.scrollTo({ top: 0 }); } function setBadge(n) { @@ -723,6 +756,52 @@ $('metadataFilterForm').addEventListener('submit', e => { e.preventDefault(); metadataOffset = 0; loadMetadata(); }); $('metadataPrev').addEventListener('click', () => { metadataOffset = Math.max(0, metadataOffset - 50); loadMetadata(); }); $('metadataNext').addEventListener('click', () => { metadataOffset += 50; loadMetadata(); }); + + /* ---------------- durable deep search ---------------- */ + let deepTimer = null; + const lines = id => $(id).value.split(/\r?\n/).map(s => s.trim()).filter(Boolean); + function deepBody() { + return { seeds: lines('deepSeeds'), modifiers: lines('deepModifiers'), maxQueries: Number($('deepMaxQueries').value), + resultsPerQuery: Number($('deepResults').value), relatedDepth: Number($('deepDepth').value), relatedTop: Number($('deepRelatedTop').value), + channels: lines('deepChannels'), playlists: lines('deepPlaylists'), delaySeconds: Number($('deepDelay').value), dailyCap: Number($('deepDailyCap').value) }; + } + if (!$('deepSeeds').value) $('deepSeeds').value = 'worship\npraise\nhymns\ngospel\npagsamba\npapuri\nmga awit ng papuri\nebanghelyo\ndayeg\npagdayeg\nmga alawiton\nmaayong balita\nadoración\nalabanza\nhimnos\nevangelio\nHillsong Worship\nElevation Worship\nBethel Music\nPlanetshakers\nDon Moen\nSandi Patty'; + if (!$('deepModifiers').value) $('deepModifiers').value = 'lyrics\nlive\nacoustic\ninstrumental\nkaraoke'; + $('deepEstimate').addEventListener('click', async () => { + const label = $('deepEstimateText'); label.textContent = 'Counting the plan…'; + try { const j = await api('/api/admin/deep-search/estimate', { method: 'POST', body: deepBody() }); + label.textContent = j.ok ? `${j.queries} unique queries · ${j.browseSources} browse sources · about ${j.estimatedMinutes} min · ${j.indexSize.toLocaleString()} indexed · ${j.dailyRemaining} daily requests remain` : j.error || 'Estimate unavailable'; + } catch { label.textContent = 'Could not reach the server.'; } + }); + $('deepSearchForm').addEventListener('submit', async e => { + e.preventDefault(); const button = $('deepStart'); button.disabled = true; + try { const j = await api('/api/admin/deep-search/start', { method: 'POST', body: deepBody() }); + $('deepEstimateText').textContent = j.ok ? 'Job queued. It will continue if this page closes.' : j.error || 'Could not start job.'; + await loadDeepSearch(); + } catch { $('deepEstimateText').textContent = 'Could not reach the server.'; } + finally { button.disabled = false; } + }); + async function loadDeepSearch() { + try { + const j = await api('/api/admin/deep-search/jobs'); if (!j.ok) throw Error(j.error || 'Could not load deep-search jobs.'); + $('deepIndexStatus').textContent = `${Number(j.indexSize).toLocaleString()} videos in the local search index.`; + $('deepJobs').innerHTML = j.jobs.length ? j.jobs.map(job => { + const active = ['queued','running','paused'].includes(job.status), progress = job.queriesTotal ? Math.round(job.queriesDone / job.queriesTotal * 100) : 0; + const eta = job.etaMs == null ? 'estimating' : job.etaMs < 60000 ? `${Math.ceil(job.etaMs / 1000)} sec` : `${Math.ceil(job.etaMs / 60000)} min`; + const controls = job.status === 'running' || job.status === 'queued' ? `` : job.status === 'paused' ? `` : ''; + return `
${esc(job.status[0].toUpperCase() + job.status.slice(1))}${job.queriesDone}/${job.queriesTotal} queries · ${job.newVideos} new videos · ETA ${eta}${controls}${active ? `` : ''}
${job.current ? `

Current: ${esc(job.current)}

` : ''}${job.errors?.length ? `
${job.errors.length} errors${job.errors.map(x => `

${esc(x.query)}: ${esc(x.error)}

`).join('')}
` : ''}
Live log (${job.log?.length || 0})
${(job.log || []).map(x => `${new Date(x.at).toLocaleTimeString()} ${esc(x.message)}`).join('\n')}
`; + }).join('') : '

No deep-search runs yet.

'; + clearTimeout(deepTimer); deepTimer = null; + if (tab === 'deep-search' && j.jobs.some(job => ['queued','running'].includes(job.status))) deepTimer = setTimeout(loadDeepSearch, 3000); + } catch (error) { $('deepIndexStatus').textContent = error.message || 'Could not load deep-search progress.'; } + } + $('deepRefresh').addEventListener('click', loadDeepSearch); + $('deepJobs').addEventListener('click', async e => { + const button = e.target.closest('[data-deep-action]'); if (!button) return; + button.disabled = true; + try { const j = await api(`/api/admin/deep-search/${encodeURIComponent(button.dataset.id)}/${button.dataset.deepAction}`, { method: 'POST', body: {} }); if (!j.ok) $('deepIndexStatus').textContent = j.error || `Could not ${button.dataset.deepAction} job.`; } + finally { await loadDeepSearch(); } + }); $('metadataTable').addEventListener('click', async e => { const btn = e.target.closest('[data-metadata-id]'); if (!btn) return; const j = await api('/api/admin/metadata/' + btn.dataset.metadataId); if (!j.ok) return; @@ -902,7 +981,7 @@ const [m, u] = await Promise.all([api('/api/admin/media'), api('/api/uploads?limit=200')]); const seen = new Set(); SONGS = []; - for (const r of (m.media || [])) { seen.add(r.id); SONGS.push({ id: r.id, title: r.title || r.id, sub: r.channel, lines: r.lyricsLines || 0 }); } + for (const r of (m.media || [])) { seen.add(r.id); SONGS.push({ id: r.id, title: r.title || r.id, sub: r.channel, lines: r.lyricsLines || 0, sourceUrl: r.sourceUrl }); } for (const r of (u.uploads || [])) if (!seen.has(r.id)) SONGS.push({ id: r.id, title: r.title, sub: r.artist || 'Upload', lines: -1 }); paintSongs(); } @@ -912,6 +991,7 @@ $('songList').innerHTML = rows.length ? rows.map((r) => `
+ ${r.sourceUrl ? `Source` : ''}
`).join('') : '
Nothing matches — paste a link above to open any YouTube song.
'; paintWhisper(); @@ -919,8 +999,17 @@ $('songFilter').addEventListener('input', paintSongs); $('songList').addEventListener('click', (e) => { const whisper = e.target.closest('[data-whisper]'); if (whisper) { startWhisper(whisper.dataset.whisper); return; } + const source = e.target.closest('[data-redownload]'); if (source) { startSourceRedownload(source); return; } const b = e.target.closest('[data-open]'); if (b) leOpen(b.dataset.open); }); + async function startSourceRedownload(button) { + button.disabled = true; const original = button.textContent; button.textContent = 'Queueing…'; + try { const result = await api(`/api/admin/media/${encodeURIComponent(button.dataset.redownload)}/redownload`, { method: 'POST', body: {} }); + button.textContent = result.ok ? 'Queued' : 'Could not queue'; + button.title = result.error || 'The server-owned download continues if this page closes.'; + } catch { button.textContent = 'Connection failed'; } + setTimeout(() => { button.textContent = original; button.disabled = false; }, 2000); + } $('recentChips').addEventListener('click', (e) => { const b = e.target.closest('[data-open]'); if (b) leOpen(b.dataset.open); }); /* ---------------- p2p ---------------- */ diff --git a/playwright.admin-deep-search.config.js b/playwright.admin-deep-search.config.js new file mode 100644 index 0000000..671b668 --- /dev/null +++ b/playwright.admin-deep-search.config.js @@ -0,0 +1,3 @@ +const { defineConfig } = require('@playwright/test'); +const classic = require('./playwright.classic.config'); +module.exports = defineConfig({ ...classic, testMatch: /admin-deep-search\.spec\.js/ }); diff --git a/server/db.js b/server/db.js index f5e394c..a3f8e4e 100644 --- a/server/db.js +++ b/server/db.js @@ -168,6 +168,15 @@ export async function initDb() { ); CREATE INDEX IF NOT EXISTS idx_collections_queue ON metadata_collections (status,created_at); + CREATE TABLE IF NOT EXISTS deep_search_jobs ( + id TEXT PRIMARY KEY, status TEXT NOT NULL, plan TEXT NOT NULL, options TEXT NOT NULL, + state TEXT NOT NULL DEFAULT '{}', error TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_deep_search_jobs ON deep_search_jobs(status,created_at); + CREATE TABLE IF NOT EXISTS deep_search_daily ( + day TEXT PRIMARY KEY, queries INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE IF NOT EXISTS lyric_transcriptions ( id TEXT PRIMARY KEY, video_id TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'queued', base_rev INTEGER NOT NULL DEFAULT 0, result TEXT, error TEXT, stage TEXT NOT NULL DEFAULT 'queued', diff --git a/server/deep-search.js b/server/deep-search.js new file mode 100644 index 0000000..5e387d7 --- /dev/null +++ b/server/deep-search.js @@ -0,0 +1,214 @@ +import { randomUUID } from 'node:crypto'; +import { db as defaultDb } from './db.js'; +import * as catalog from './video-catalog.js'; +import * as innertube from './innertube.js'; +import * as searchCache from './search-cache.js'; + +const MAX_SEEDS = 200, MAX_PLAN = 500, MAX_ERRORS = 50; +const clean = value => String(value || '').replace(/[\u0000-\u001f\u007f]/g, ' ').trim().slice(0, 200); +const fold = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase().replace(/\s+/g, ' ').trim(); +const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } }; +const sleepDefault = ms => new Promise(resolve => setTimeout(resolve, ms)); + +export const DEFAULT_PRESETS = [ + 'worship', 'praise', 'hymns', 'gospel', 'pagsamba', 'papuri', 'mga awit ng papuri', 'ebanghelyo', + 'dayeg', 'pagdayeg', 'mga alawiton', 'maayong balita', 'adoración', 'alabanza', 'himnos', 'evangelio', + 'Hillsong Worship', 'Elevation Worship', 'Bethel Music', 'Planetshakers', 'Don Moen', 'Sandi Patty', +]; +export const DEFAULT_MODIFIERS = ['lyrics', 'live', 'acoustic', 'instrumental', 'karaoke']; + +export function expandQueryPlan({ seeds = [], modifiers = [], maxQueries = 80 } = {}) { + const seedList = (Array.isArray(seeds) ? seeds : String(seeds).split(/\r?\n/)).map(clean).filter(Boolean).slice(0, MAX_SEEDS); + const mods = (Array.isArray(modifiers) ? modifiers : String(modifiers).split(/[,\r\n]+/)).map(clean).filter(Boolean).slice(0, 30); + const cap = Math.max(1, Math.min(MAX_PLAN, Number(maxQueries) || 1)); + const out = [], seen = new Set(); + for (const seed of seedList) for (const modifier of ['', ...mods]) { + const query = clean(modifier ? `${seed} ${modifier}` : seed), key = fold(query); + if (key && !seen.has(key)) { seen.add(key); out.push(query); if (out.length >= cap) return out; } + } + return out; +} + +function publicJob(row, indexSize = 0) { + const state = parse(row.state); + const done = Number(state.queriesDone) || 0, total = JSON.parse(row.plan || '[]').length; + const elapsed = Math.max(0, Date.now() - Number(row.created_at)); + const etaMs = done ? Math.max(0, Math.round(elapsed / done * (total - done))) : null; + return { id: row.id, status: row.status, plan: JSON.parse(row.plan || '[]'), options: parse(row.options), + queriesDone: done, queriesTotal: total, current: state.current || '', newVideos: Number(state.newVideos) || 0, + totalIndexSize: indexSize, errors: state.errors || [], log: state.log || [], etaMs, + createdAt: Number(row.created_at), updatedAt: Number(row.updated_at), error: row.error || null }; +} + +export function createDeepSearchService({ db = defaultDb, searchDeep = innertube.searchDeep, related = innertube.related, + browseUploads = innertube.browseUploads, ingest = catalog.ingest, putSearch = searchCache.put, + sleep = sleepDefault, random = Math.random, now = Date.now, autoStart = true } = {}) { + let worker = null; + let timer = null; + const log = (state, message) => { state.log ||= []; state.log.push({ at: now(), message: String(message).slice(0, 240) }); if (state.log.length > 100) state.log.splice(0, state.log.length - 100); }; + async function indexSize() { return Number((await db.execute('SELECT COUNT(*) AS n FROM video_meta')).rows[0]?.n || 0); } + async function getJob(id) { return (await db.execute({ sql: 'SELECT * FROM deep_search_jobs WHERE id=?', args: [id] })).rows[0] || null; } + async function checkpoint(job, state, status = 'running', error = null) { + const r = await db.execute({ sql: "UPDATE deep_search_jobs SET state=?,status=?,error=?,updated_at=? WHERE id=? AND status='running'", args: [JSON.stringify(state), status, error, now(), job.id] }); + if (!r.rowsAffected) { const stop = new Error('Deep search paused or cancelled.'); stop.name = 'DeepSearchStopped'; throw stop; } + } + async function reserveDaily(cap) { + const day = new Date(now()).toISOString().slice(0, 10); + const r = await db.execute({ sql: `INSERT INTO deep_search_daily(day,queries) VALUES(?,1) + ON CONFLICT(day) DO UPDATE SET queries=queries+1 WHERE queries c?.id).map(c => [c.id, c])).values()]; + if (unique.length) { + const validIds = unique.filter(c => /^[\w-]{11}$/.test(c.id)).map(c => c.id); + const existing = validIds.length ? (await db.execute({ sql: `SELECT id FROM video_meta WHERE id IN (${validIds.map(() => '?').join(',')})`, args: validIds })).rows : []; + const present = new Set(existing.map(row => row.id)); + await ingest(unique, 'deep'); + state.newVideos += validIds.filter(id => !present.has(id)).length; + } + await putSearch(query, (cacheCards || unique).slice(0, 200)); + } + async function requireDailyBudget() { + const used = await reserveDaily(Math.max(1, Number(options.dailyCap) || 1000)); + if (used !== null) return; + state.dailyPausedDay = new Date(now()).toISOString().slice(0, 10); + log(state, 'Daily query cap reached; job will resume tomorrow.'); + await checkpoint(job, state, 'paused'); + const stop = new Error('Daily cap reached.'); stop.name = 'DeepSearchStopped'; throw stop; + } + try { + for (let qi = state.queriesDone; qi < plan.length; qi++) { + const query = plan[qi]; + if (state.current !== query) state.currentIds = []; + state.current = query; await checkpoint(job, state); + await requireDailyBudget(); + await checkpoint(job, state); + let cards, lastError; + for (let attempt = 0; attempt < 3; attempt++) { + try { + cards = await searchDeep(query, { limit: Math.max(1, Math.min(200, Number(options.resultsPerQuery) || 50)), maxPages: 14, timeoutMs: 30000, + pageDelayMs: Math.max(0, Number(options.delayMs) || 0), jitterMs: Math.max(0, Number(options.jitterMs) || 0), + onPage: async pageCards => { + await checkpoint(job, state); + const saved = new Set(state.currentIds || []); + const fresh = pageCards.filter(card => !saved.has(card.id)); + await saveCards(query, fresh, pageCards); + state.currentIds = pageCards.map(card => card.id); + await checkpoint(job, state); + } }); + await checkpoint(job, state); + lastError = null; break; + } catch (error) { if (error.name === 'DeepSearchStopped') throw error; lastError = error; if (attempt < 2) await sleep(Math.min(60000, (Number(options.delayMs) || 10000) * 2 ** attempt + Math.floor(random() * 3000))); } + } + if (lastError) { state.errors.push({ query, error: String(lastError.message).slice(0, 200) }); if (state.errors.length > MAX_ERRORS) state.errors.shift(); log(state, `Search failed: ${query}`); } + else { + const saved = new Set(state.currentIds || []); + const remaining = (cards || []).filter(card => !saved.has(card.id)); + await saveCards(query, remaining, cards || []); + state.currentIds = (cards || []).map(card => card.id); + const depth = Math.max(0, Math.min(2, Number(options.relatedDepth) || 0)); + if (depth) { + let frontier = (cards || []).slice(0, Math.max(1, Math.min(10, Number(options.relatedTop) || 3))).map(c => ({ id: c.id, depth: 1 })); + for (let level = 1; level <= depth && frontier.length; level++) { + const next = []; + for (const item of frontier) { + if (state.relatedSeen.includes(item.id)) continue; + state.relatedSeen.push(item.id); + try { + await sleep(Math.max(0, Number(options.delayMs) || 10000) + Math.floor(random() * (Number(options.jitterMs) || 3000))); + await checkpoint(job, state); + await requireDailyBudget(); + const recs = await related(item.id); await checkpoint(job, state); + await saveCards(`${query} related ${item.id}`, recs); + if (level < depth) next.push(...recs.slice(0, 2).map(c => ({ id: c.id, depth: level + 1 }))); + } catch (error) { if (error.name === 'DeepSearchStopped') throw error; state.errors.push({ query: `related:${item.id}`, error: String(error.message).slice(0, 200) }); } + } + frontier = next; + } + } + log(state, `Saved ${cards?.length || 0} results for “${query}”.`); + } + state.queriesDone = qi + 1; state.current = ''; state.currentIds = []; await checkpoint(job, state); + if (qi + 1 < plan.length) await sleep(Math.max(0, Number(options.delayMs) || 10000) + Math.floor(random() * (Number(options.jitterMs) || 3000))); + } + for (const source of [...(options.channels || []), ...(options.playlists || [])].slice(0, 50)) { + state.current = `browse ${source}`; await checkpoint(job, state); + try { await requireDailyBudget(); await checkpoint(job, state); const cards = await browseUploads(source, { limit: Math.max(1, Math.min(200, Number(options.resultsPerQuery) || 50)), pageDelayMs: options.delayMs, jitterMs: options.jitterMs }); await checkpoint(job, state); await saveCards(state.current, cards); log(state, `Browsed ${cards.length} videos from ${source}.`); } + catch (error) { if (error.name === 'DeepSearchStopped') throw error; state.errors.push({ query: state.current, error: String(error.message).slice(0, 200) }); } + await checkpoint(job, state); + } + state.current = ''; log(state, 'Deep search complete.'); await checkpoint(job, state, 'complete'); + } catch (error) { + if (error.name !== 'DeepSearchStopped') { + state.errors.push({ query: state.current, error: String(error.message).slice(0, 200) }); + await db.execute({ sql: "UPDATE deep_search_jobs SET state=?,status='failed',error=?,updated_at=? WHERE id=? AND status='running'", args: [JSON.stringify(state), String(error.message).slice(0, 300), now(), job.id] }); + } + } + } + async function runNext() { + if (worker) return worker; + worker = (async () => { + // Recover jobs interrupted by process restart; the current query is safe to repeat. + await db.execute({ sql: "UPDATE deep_search_jobs SET status='queued',updated_at=? WHERE status='running'", args: [now()] }); + const today = new Date(now()).toISOString().slice(0, 10); + const paused = (await db.execute("SELECT id,state FROM deep_search_jobs WHERE status='paused'")).rows; + for (const row of paused) if (parse(row.state).dailyPausedDay && parse(row.state).dailyPausedDay < today) + await db.execute({ sql: "UPDATE deep_search_jobs SET status='queued',updated_at=? WHERE id=? AND status='paused'", args: [now(), row.id] }); + const job = (await db.execute({ sql: "SELECT * FROM deep_search_jobs WHERE status='queued' ORDER BY created_at LIMIT 1" })).rows[0]; + if (job) { + await db.execute({ sql: "UPDATE deep_search_jobs SET status='running',updated_at=? WHERE id=? AND status='queued'", args: [now(), job.id] }); + const current = await getJob(job.id); if (current?.status === 'running') await run(current); + } + return !!job; + })().finally(() => { worker = null; }); + return worker; + } + const kick = () => runNext().catch(error => console.error('[deep search]', error.message)); + function start() { if (!timer) { timer = setInterval(kick, 15000); timer.unref?.(); } kick(); } + if (autoStart) start(); + async function estimate(body) { + const plan = expandQueryPlan(body); + const browseSources = [...(Array.isArray(body?.channels) ? body.channels : []), ...(Array.isArray(body?.playlists) ? body.playlists : [])].map(clean).filter(Boolean).slice(0, 50).length; + const operations = plan.length + browseSources + (Number(body?.relatedDepth) > 0 ? Math.min(plan.length * (Number(body?.relatedTop) || 3) * Math.min(2, Number(body.relatedDepth)), 100) : 0); + const today = new Date(now()).toISOString().slice(0, 10); + const used = Number((await db.execute({ sql: 'SELECT queries FROM deep_search_daily WHERE day=?', args: [today] })).rows[0]?.queries || 0); + return { plan, queries: plan.length, browseSources, estimatedOperations: operations, + estimatedMinutes: Math.ceil(operations * Math.max(1, Number(body?.delaySeconds) || 10) / 60), + indexSize: await indexSize(), dailyUsed: used, dailyRemaining: Math.max(0, (Number(body?.dailyCap) || 1000) - used) }; + } + function register(app, adminAuth) { + app.post('/api/admin/deep-search/estimate', adminAuth, async c => { let body = {}; try { body = await c.req.json(); } catch {} return c.json({ ok: true, ...(await estimate(body)) }); }); + app.get('/api/admin/deep-search/jobs', adminAuth, async c => c.json({ ok: true, indexSize: await indexSize(), jobs: (await db.execute('SELECT * FROM deep_search_jobs ORDER BY created_at DESC LIMIT 30')).rows.map(row => publicJob(row)) })); + app.post('/api/admin/deep-search/start', adminAuth, async c => { + let body; try { body = await c.req.json(); } catch { return c.json({ ok: false, error: 'Invalid JSON.' }, 400); } + const plan = expandQueryPlan(body); if (!plan.length) return c.json({ ok: false, error: 'Add at least one seed term.' }, 400); + const active = (await db.execute("SELECT id FROM deep_search_jobs WHERE status IN ('queued','running','paused') LIMIT 1")).rows[0]; + if (active) return c.json({ ok: false, error: 'A deep-search job is already active.' }, 409); + const options = { resultsPerQuery: Math.max(1, Math.min(200, Number(body.resultsPerQuery) || 50)), relatedDepth: Math.max(0, Math.min(2, Number(body.relatedDepth) || 0)), relatedTop: Math.max(1, Math.min(10, Number(body.relatedTop) || 3)), delayMs: Math.max(1000, Math.min(300000, (Number(body.delaySeconds) || 10) * 1000)), jitterMs: Math.max(0, Math.min(300000, (Number(body.jitterSeconds) || 3) * 1000)), dailyCap: Math.max(1, Math.min(10000, Number(body.dailyCap) || 1000)), channels: (Array.isArray(body.channels) ? body.channels : []).map(clean).filter(Boolean).slice(0, 25), playlists: (Array.isArray(body.playlists) ? body.playlists : []).map(clean).filter(Boolean).slice(0, 25) }; + const id = randomUUID(), time = now(); + await db.execute({ sql: "INSERT INTO deep_search_jobs(id,status,plan,options,state,created_at,updated_at) VALUES(?,'queued',?,?,?,?,?)", args: [id, JSON.stringify(plan), JSON.stringify(options), JSON.stringify({ queriesDone: 0, newVideos: 0, errors: [], log: [] }), time, time] }); + if (autoStart) kick(); return c.json({ ok: true, id }, 202); + }); + for (const action of ['pause', 'resume', 'cancel']) app.post(`/api/admin/deep-search/:id/${action}`, adminAuth, async c => { + const id = c.req.param('id'); + const allowed = action === 'pause' ? ['queued', 'running'] : action === 'resume' ? ['paused'] : ['queued', 'running', 'paused']; + const target = action === 'pause' ? 'paused' : action === 'resume' ? 'queued' : 'cancelled'; + const marks = allowed.map(() => '?').join(','); + const r = await db.execute({ sql: `UPDATE deep_search_jobs SET status=?,updated_at=? WHERE id=? AND status IN (${marks})`, args: [target, now(), id, ...allowed] }); + if (action === 'resume' && r.rowsAffected && autoStart) kick(); + return c.json({ ok: !!r.rowsAffected, status: r.rowsAffected ? target : undefined }, r.rowsAffected ? 200 : 409); + }); + return { estimate, runNext, kick, start }; + } + return { estimate, runNext, kick, start, register }; +} + +export function registerDeepSearchRoutes(app, options = {}) { + const service = createDeepSearchService(options); + service.register(app, options.adminAuth); + return service; +} diff --git a/server/deep-search.test.js b/server/deep-search.test.js new file mode 100644 index 0000000..b9710ef --- /dev/null +++ b/server/deep-search.test.js @@ -0,0 +1,122 @@ +import { expect, test, beforeAll } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Hono } from 'hono'; + +const root = mkdtempSync(join(tmpdir(), 'ytp-deep-search-')); +process.env.DB_PATH = join(root, 'test.db'); +const { initDb, db } = await import('./db.js'); +const catalog = await import('./video-catalog.js'); +const searchCache = await import('./search-cache.js'); +const { createDeepSearchService, expandQueryPlan } = await import('./deep-search.js'); +const auth = async (_c, next) => next(); +const sample = { id: 'd33psrch001', title: 'Worship in Cebuano', channel: 'Hope Choir' }; + +beforeAll(async () => { await initDb(); }); + +function makeApp(service) { + const app = new Hono(); service.register(app, auth); return app; +} +async function request(app, path, method = 'GET', body) { + const response = await app.request(path, { method, headers: body ? { 'content-type': 'application/json' } : {}, body: body ? JSON.stringify(body) : undefined }); + return { response, json: await response.json() }; +} +const cleanDb = async () => { + await db.batch(['DELETE FROM deep_search_jobs', 'DELETE FROM deep_search_daily', 'DELETE FROM video_meta', 'DELETE FROM video_meta_sources', 'DELETE FROM search_cache']); +}; + +test.serial('query plan expands modifiers, folds accents and deduplicates', () => { + expect(expandQueryPlan({ seeds: ['Adoración', 'adoracion', ' Gospel '], modifiers: ['live', 'LIVE'], maxQueries: 30 })) + .toEqual(['Adoración', 'Adoración live', 'Gospel', 'Gospel live']); + expect(expandQueryPlan({ seeds: ['one', 'two'], modifiers: ['live'], maxQueries: 2 })).toHaveLength(2); +}); + +test.serial('estimate and background job ingest to catalog, source and search cache', async () => { + await cleanDb(); + const service = createDeepSearchService({ db, ingest: catalog.ingest, putSearch: searchCache.put, + searchDeep: async () => [sample], sleep: async () => {}, autoStart: false }); + const app = makeApp(service); + const estimate = await request(app, '/api/admin/deep-search/estimate', 'POST', { seeds: ['worship'], modifiers: ['live','lyrics'], maxQueries: 4 }); + expect(estimate.json.plan).toEqual(['worship', 'worship live', 'worship lyrics']); + const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], modifiers: [], maxQueries: 1, delaySeconds: 1 }); + expect(created.response.status).toBe(202); + await service.runNext(); + const jobs = await request(app, '/api/admin/deep-search/jobs'); + expect(jobs.json.jobs[0].status).toBe('complete'); + expect(jobs.json.jobs[0].newVideos).toBe(1); + const row = (await db.execute({ sql: 'SELECT source FROM video_meta_sources WHERE video_id=?', args: [sample.id] })).rows[0]; + expect(row.source).toBe('deep'); + expect((await searchCache.get('worship')).results[0].id).toBe(sample.id); +}); + +test.serial('pause during upstream call does not ingest; resume continues from the persisted query', async () => { + await cleanDb(); let begin, release, calls = 0; + const began = new Promise(resolve => { begin = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + const service = createDeepSearchService({ db, ingest: catalog.ingest, putSearch: searchCache.put, autoStart: false, + sleep: async () => {}, searchDeep: async () => { calls++; if (calls === 1) { begin(); await gate; } return [sample]; } }); + const app = makeApp(service); + const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 }); + const work = service.runNext(); await began; + await request(app, `/api/admin/deep-search/${created.json.id}/pause`, 'POST', {}); + release(); await work; + expect((await db.execute({ sql: 'SELECT COUNT(*) AS n FROM video_meta WHERE id=?', args: [sample.id] })).rows[0].n).toBe(0); + let jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('paused'); + await request(app, `/api/admin/deep-search/${created.json.id}/resume`, 'POST', {}); + await service.runNext(); + jobs = await request(app, '/api/admin/deep-search/jobs'); + expect(jobs.json.jobs[0].status).toBe('complete'); expect(calls).toBe(2); +}); + +test.serial('cancelled in-flight job does not write its late result', async () => { + await cleanDb(); let begin, release; + const began = new Promise(resolve => { begin = resolve; }), gate = new Promise(resolve => { release = resolve; }); + const service = createDeepSearchService({ db, autoStart: false, sleep: async () => {}, + searchDeep: async () => { begin(); await gate; return [sample]; }, ingest: catalog.ingest, putSearch: searchCache.put }); + const app = makeApp(service); + const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 }); + const work = service.runNext(); await began; + await request(app, `/api/admin/deep-search/${created.json.id}/cancel`, 'POST', {}); release(); await work; + expect((await db.execute({ sql: 'SELECT COUNT(*) AS n FROM video_meta WHERE id=?', args: [sample.id] })).rows[0].n).toBe(0); + const jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('cancelled'); +}); + +test.serial('page checkpoint survives a retry without double-counting its videos', async () => { + await cleanDb(); let calls = 0; + const service = createDeepSearchService({ db, autoStart: false, sleep: async () => {}, ingest: catalog.ingest, putSearch: searchCache.put, + searchDeep: async (_query, { onPage }) => { + calls++; + await onPage([sample]); + if (calls === 1) throw Error('connection reset after first page'); + return [sample]; + } }); + const app = makeApp(service); + await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 }); + await service.runNext(); + const row = (await db.execute({ sql: 'SELECT seen FROM video_meta WHERE id=?', args: [sample.id] })).rows[0]; + expect(Number(row.seen)).toBe(1); +}); + +test.serial('daily cap pauses and worker resumes it on the next UTC day', async () => { + await cleanDb(); let time = Date.parse('2026-10-14T23:59:00Z'); + const service = createDeepSearchService({ db, now: () => time, autoStart: false, sleep: async () => {}, + searchDeep: async query => [{ ...sample, id: query === 'worship' ? 'd33psrch001' : 'd33psrch002' }], + ingest: catalog.ingest, putSearch: searchCache.put }); + const app = makeApp(service); + const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship','praise'], maxQueries: 2, delaySeconds: 1, dailyCap: 1 }); + await service.runNext(); + let jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('paused'); + time += 120000; await service.runNext(); jobs = await request(app, '/api/admin/deep-search/jobs'); + expect(jobs.json.jobs[0].status).toBe('complete'); expect(jobs.json.jobs[0].queriesDone).toBe(2); +}); + +test.serial('only one queued/running job is accepted', async () => { + await cleanDb(); + const service = createDeepSearchService({ db, autoStart: false }); const app = makeApp(service); + const body = { seeds: ['worship'], maxQueries: 1 }; + expect((await request(app, '/api/admin/deep-search/start', 'POST', body)).response.status).toBe(202); + expect((await request(app, '/api/admin/deep-search/start', 'POST', body)).response.status).toBe(409); +}); + +test.serial('cleanup test database', () => { rmSync(root, { recursive: true, force: true }); }); diff --git a/server/innertube.js b/server/innertube.js index 356479c..cc9dbf9 100644 --- a/server/innertube.js +++ b/server/innertube.js @@ -79,7 +79,7 @@ export async function search(q, { fetchImpl = fetch, timeoutMs = 6000 } = {}) { // are a chain, so they are fetched one after another; every page is optional — // a failure part-way returns what was gathered. `onPage(cards)` lets a caller // save progress. -export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs = 6000, maxPages = 14, onPage } = {}) { +export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs = 6000, maxPages = 14, onPage, pageDelayMs = 0, jitterMs = 0 } = {}) { const post = async (body) => { const res = await fetchImpl('https://www.youtube.com/youtubei/v1/search?prettyPrint=false', { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -93,19 +93,92 @@ export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs const all = []; const add = (cards) => { for (const c of cards) if (!seen.has(c.id)) { seen.add(c.id); all.push(c); } }; add(first.cards); - if (onPage) onPage(all.slice()); + if (onPage) await onPage(all.slice()); let next = first.next; for (let i = 0; i < maxPages && next && all.length < limit; i++) { + if (pageDelayMs > 0 || jitterMs > 0) await new Promise(resolve => setTimeout(resolve, Math.max(0, pageDelayMs + Math.random() * jitterMs))); let page; try { page = parseContinuation(await post({ continuation: next })); } catch { break; } if (!page.cards.length) break; add(page.cards); - if (onPage) onPage(all.slice()); + if (onPage) await onPage(all.slice()); next = page.next; } return all.slice(0, limit); } +function browseToken(value) { + if (!value || typeof value !== 'object') return ''; + const token = value.continuationItemRenderer?.continuationEndpoint?.continuationCommand?.token; + if (token) return token; + for (const child of Object.values(value)) { + if (Array.isArray(child)) { for (const item of child) { const found = browseToken(item); if (found) return found; } } + else { const found = browseToken(child); if (found) return found; } + } + return ''; +} + +export function parseBrowseVideos(json) { + const cards = new Map(); + function visit(value) { + if (!value || typeof value !== 'object') return; + const video = value.videoRenderer || value.gridVideoRenderer || value.richItemRenderer?.content?.videoRenderer; + const id = video?.videoId; + const title = video?.title?.runs?.map(run => run.text).join('') || video?.title?.simpleText || ''; + if (id && /^[\w-]{11}$/.test(id) && title) { + const owner = video.ownerText?.runs?.[0] || video.shortBylineText?.runs?.[0] || {}; + const browse = owner.navigationEndpoint?.browseEndpoint || {}; + cards.set(id, { id, url: `https://www.youtube.com/watch?v=${id}`, title, channel: owner.text || '', + channelId: browse.browseId || '', channelUrl: browse.canonicalBaseUrl ? `https://www.youtube.com${browse.canonicalBaseUrl}` : '', + duration: lengthToSeconds(video.lengthText?.simpleText), thumbnail: `https://i.ytimg.com/vi/${id}/mqdefault.jpg` }); + return; + } + for (const child of Object.values(value)) if (Array.isArray(child)) child.forEach(visit); else visit(child); + } + visit(json); + return [...cards.values()]; +} + +// YouTube browse continuation for channel uploads and playlists. Channel IDs +// are translated to their uploads playlist (UU…), while playlist URLs use VL. +export async function browseUploads(source, { limit = 200, fetchImpl = fetch, timeoutMs = 8000, maxPages = 12, pageDelayMs = 0, jitterMs = 0 } = {}) { + let id = String(source || '').trim(); + let handle = ''; + try { + const url = new URL(id); + id = url.searchParams.get('list') || url.pathname.match(/\/channel\/([^/]+)/)?.[1] || id; + handle = url.pathname.match(/\/@([^/]+)/)?.[1] || ''; + } catch { /* raw channel or playlist id */ } + if (handle) { + const response = await fetchImpl(`https://www.youtube.com/@${encodeURIComponent(handle)}`, { signal: AbortSignal.timeout(timeoutMs) }); + if (!response.ok) throw new Error(`channel lookup: HTTP ${response.status}`); + const html = await response.text(); + id = html.match(/"browseId"\s*:\s*"(UC[\w-]{22})"/)?.[1] || ''; + } + let browseId; + if (/^UC[\w-]{22}$/.test(id)) browseId = `UU${id.slice(2)}`; + else if (/^(?:PL|UU|LL|FL|OL)[\w-]{10,}$/.test(id)) browseId = id.startsWith('UU') || id.startsWith('LL') || id.startsWith('FL') || id.startsWith('OL') ? id : `VL${id}`; + else throw new Error('Expected a YouTube channel ID or playlist ID/URL.'); + const post = async body => { + const response = await fetchImpl('https://www.youtube.com/youtubei/v1/browse?prettyPrint=false', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ context: { client: CLIENT }, ...body }), signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) throw new Error(`innertube browse: HTTP ${response.status}`); + return response.json(); + }; + let page = await post({ browseId }); + const seen = new Set(), cards = []; + for (let i = 0; i < maxPages && cards.length < limit; i++) { + for (const card of parseBrowseVideos(page)) if (!seen.has(card.id)) { seen.add(card.id); cards.push(card); } + const token = browseToken(page); + if (!token || cards.length >= limit) break; + if (pageDelayMs > 0 || jitterMs > 0) await new Promise(resolve => setTimeout(resolve, Math.max(0, pageDelayMs + Math.random() * jitterMs))); + page = await post({ continuation: token }); + } + return cards.slice(0, limit); +} + // Watch-next recommendations use compact renderers or the newer lockup model. // Traverse only the recommendation contents, never player configuration/ads. export function parseRelated(json) { diff --git a/server/innertube.test.js b/server/innertube.test.js index da1011c..22ac50b 100644 --- a/server/innertube.test.js +++ b/server/innertube.test.js @@ -7,7 +7,7 @@ test('parses video renderers into slim cards', () => { const r = parseSearch(fixture); expect(r.length).toBe(4); expect(r[0]).toEqual({ - id: 'nQWFzMvCfLE', title: 'What A Beautiful Name - Hillsong Worship', channel: 'Hillsong Worship', + id: 'nQWFzMvCfLE', url: 'https://www.youtube.com/watch?v=nQWFzMvCfLE', title: 'What A Beautiful Name - Hillsong Worship', channel: 'Hillsong Worship', channelId: 'UC4q12NoPNySbVqwpw4iO5Vg', channelUrl: 'https://www.youtube.com/channel/UC4q12NoPNySbVqwpw4iO5Vg', duration: 343, thumbnail: 'https://i.ytimg.com/vi/nQWFzMvCfLE/mqdefault.jpg', }); @@ -60,3 +60,18 @@ test('related HTTP failures and unsupported shapes throw for search fallback', a expect(() => parseRelated({ error: {} })).toThrow('response shape'); await expect(related('aaaaaaaaaaa', { fetchImpl: async () => ({ ok: false, status: 429 }) })).rejects.toThrow('429'); }); + +test('channel and playlist browse parses uploads and resolves handle URLs', async () => { + const { browseUploads, parseBrowseVideos } = await import('./innertube.js'); + const renderer = { videoRenderer: { videoId: 'abcdefghijk', title: { simpleText: 'Upload' }, ownerText: { runs: [{ text: 'Choir', navigationEndpoint: { browseEndpoint: { browseId: 'UC1234567890123456789012', canonicalBaseUrl: '/@choir' } } }] }, lengthText: { simpleText: '2:10' } } }; + expect(parseBrowseVideos({ contents: [{ gridVideoRenderer: { videoId: 'abcdefghijk', title: { simpleText: 'Upload' } } }] })[0].title).toBe('Upload'); + const calls = []; + const fetchImpl = async (url, init = {}) => { + calls.push([String(url), init.body && JSON.parse(init.body)]); + if (String(url).includes('/@choir')) return new Response('"browseId":"UC1234567890123456789012"'); + return Response.json({ contents: [{ gridVideoRenderer: renderer.videoRenderer }] }); + }; + const result = await browseUploads('https://www.youtube.com/@choir', { fetchImpl, limit: 5 }); + expect(result[0].id).toBe('abcdefghijk'); + expect(calls[1][1].browseId).toBe('UU1234567890123456789012'); +}); diff --git a/server/package.json b/server/package.json index a4e37cf..c144a43 100644 --- a/server/package.json +++ b/server/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "bun server.js", "dev": "bun --hot server.js", - "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js" + "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js" }, "dependencies": { "@hono/node-server": "^1.14.0", diff --git a/server/search-cache.test.js b/server/search-cache.test.js index 7d25f50..3322f98 100644 --- a/server/search-cache.test.js +++ b/server/search-cache.test.js @@ -1,8 +1,13 @@ -import { test, expect } from 'bun:test'; -import * as sc from './search-cache.js'; -import { initDb } from './db.js'; - +import { test, expect, afterAll } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +const root = mkdtempSync(join(tmpdir(), 'ytp-search-cache-')); +process.env.DB_PATH = join(root, 'test.db'); +const sc = await import('./search-cache.js'); +const { initDb } = await import('./db.js'); await initDb(); +afterAll(() => rmSync(root, { recursive: true, force: true })); const cards = (n) => Array.from({ length: n }, (_, i) => ({ id: 'v' + i, title: 't' + i })); test('put/get round-trips, case-insensitive, capped at 200', async () => { diff --git a/server/server.js b/server/server.js index ec72915..376ec94 100644 --- a/server/server.js +++ b/server/server.js @@ -68,6 +68,7 @@ import { registerCatalogRoutes } from './recommendations.js'; import QRCode from 'qrcode'; import { createYtdlpPool } from './ytdlp-pool.js'; import { createWarmQueue } from './warm-queue.js'; +import { registerDeepSearchRoutes } from './deep-search.js'; import { dirname, join as pathJoin } from 'node:path'; // A media proxy must not die because one client's stream hit an edge case @@ -1113,10 +1114,11 @@ async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal, sourceUrl const tmp = `${tmpdir()}/${tmpBase}.mp4`; const ctl = new AbortController(); entry = { waiters: 0, tmpBase, ctl }; - entry.promise = withSaveSlot(() => { + entry.promise = withSaveSlot(async () => { if (ctl.signal.aborted) throw new Error('save cancelled'); + const url = sourceUrl || await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`; return runYtdlpResilient([ - sourceUrl || await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`, + url, '--no-warnings', '--no-playlist', ...formatArgs, '--limit-rate', DOWNLOAD_RATE, @@ -1186,14 +1188,16 @@ async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal, cachedSrc try { // 1) Grab the full source (video+audio merged) so ffmpeg has both streams // — unless the server cache already holds it. - if (!cachedSrc) await withSaveSlot(() => runYtdlpResilient([ - await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`, - '--no-warnings', '--no-playlist', - '-f', SAVE_MUX_FORMAT, - '--merge-output-format', 'mp4', - '--limit-rate', DOWNLOAD_RATE, - '-o', srcTmp, - ], { signal })); + if (!cachedSrc) await withSaveSlot(async () => { + const url = await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`; + return runYtdlpResilient([ + url, '--no-warnings', '--no-playlist', + '-f', SAVE_MUX_FORMAT, + '--merge-output-format', 'mp4', + '--limit-rate', DOWNLOAD_RATE, + '-o', srcTmp, + ], { signal }); + }); // 2) Trim + concat the keep segments into the final custom video. await runFfmpeg([ '-y', '-hide_banner', '-loglevel', 'error', @@ -2295,6 +2299,8 @@ app.get('/api/remote/qr/:code', async (c) => { // ============================================================================ const notes = registerNoteRoutes(app, { db: notesDb, + catalog: videoCatalog, + media, getProfile, profileNameRe: PROFILE_NAME_RE, runYtdlp: runYtdlpResilient, @@ -2308,6 +2314,9 @@ const notes = registerNoteRoutes(app, { { label: 'Database', path: dirname(process.env.DB_PATH || './data/ytplayer.db') }, ] }, }); +const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, autoStart: false, + searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads, + ingest: collectVideoMetadata, putSearch: searchCacheDb.put }); // ============================================================================ // Uploads — the server's own searchable media library (see uploads.js) @@ -2519,6 +2528,7 @@ app.get('/*', indexHtml); // ============================================================================ async function main() { await initDb(); + deepSearch.start(); startThumbnails(); await p2pDb.initP2pSchema(); console.log(`[ytplayer] DB ready`); diff --git a/server/video-catalog.js b/server/video-catalog.js index 4f32acf..5424d3e 100644 --- a/server/video-catalog.js +++ b/server/video-catalog.js @@ -3,7 +3,7 @@ import { db } from './db.js'; import { mergeMissing } from './media-meta-core.js'; const VIDEO_ID = /^[\w-]{11}$/; -const SOURCES = new Set(['search', 'search-cache', 'client-search', 'channel', 'streams', 'playlist', 'profile', 'sync', 'related', 'backfill', 'collector', 'device-play']); +const SOURCES = new Set(['search', 'search-cache', 'client-search', 'channel', 'streams', 'playlist', 'profile', 'sync', 'related', 'backfill', 'collector', 'deep', 'device-play']); const text = (v, n = 300) => typeof v === 'string' ? v.trim().slice(0, n) : ''; const positive = v => Number.isFinite(Number(v)) && Number(v) > 0 ? Number(v) : 0; const canonicalThumb = id => `https://i.ytimg.com/vi/${id}/hqdefault.jpg`; diff --git a/tests/admin-deep-search.spec.js b/tests/admin-deep-search.spec.js new file mode 100644 index 0000000..7da7de2 --- /dev/null +++ b/tests/admin-deep-search.spec.js @@ -0,0 +1,38 @@ +const { test, expect } = require('@playwright/test'); + +test('admin deep-search tab estimates, starts, controls and shows persisted progress', async ({ page }) => { + page.on('pageerror', error => console.error('pageerror:', error.message)); + const state = { jobs: [], events: [] }; + await page.route('**/api/**', async route => { + const req = route.request(), url = new URL(req.url()); + let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true }; + else if (url.pathname === '/api/admin/media') body = { ok: true, media: [] }; + else if (url.pathname === '/api/uploads') body = { ok: true, uploads: [] }; + else if (url.pathname === '/api/admin/deep-search/estimate') body = { ok: true, plan: ['worship', 'worship live'], queries: 2, estimatedMinutes: 1, indexSize: 1240, dailyRemaining: 999 }; + else if (url.pathname === '/api/admin/deep-search/start') { + state.events.push('start'); state.jobs = [{ id: 'deep-1', status: 'running', plan: ['worship', 'worship live'], options: {}, queriesDone: 1, queriesTotal: 2, current: 'worship live', newVideos: 24, totalIndexSize: 1264, etaMs: 12000, errors: [], log: [{ at: Date.now(), message: 'Saved 24 results' }] }]; + body = { ok: true, id: 'deep-1' }; + } else if (url.pathname === '/api/admin/deep-search/jobs') body = { ok: true, indexSize: 1264, jobs: state.jobs }; + else if (/\/api\/admin\/deep-search\/deep-1\/(pause|resume|cancel)$/.test(url.pathname)) { + const action = url.pathname.split('/').at(-1); state.events.push(action); + state.jobs[0].status = action === 'pause' ? 'paused' : action === 'resume' ? 'running' : 'cancelled'; + body = { ok: true, status: state.jobs[0].status }; + } + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.goto('/admin.html#deep-search'); + await expect(page.locator('#deepSearchForm')).toBeVisible(); + await expect(page.locator('#deepSeeds')).toHaveValue(/worship/); + await page.locator('#deepEstimate').click(); + await expect(page.locator('#deepEstimateText')).toContainText('2 unique queries'); + await page.locator('#deepStart').click(); + await expect(page.locator('.deep-job')).toContainText('1/2 queries'); + await page.locator('[data-deep-action="pause"]').click(); + await expect(page.locator('.deep-job')).toContainText('Paused'); + await page.locator('[data-deep-action="resume"]').click(); + await expect(page.locator('.deep-job')).toContainText('Running'); + await page.locator('[data-deep-action="cancel"]').click(); + await expect(page.locator('.deep-job')).toContainText('Cancelled'); + expect(state.events).toEqual(['start', 'pause', 'resume', 'cancel']); +});