Add resumable admin deep search
This commit is contained in:
Jonathan Sykes
@@ -280,6 +280,14 @@
|
||||
.collection-job p { margin: 4px 0; color: var(--text-2); }
|
||||
#metadataDetail { white-space: pre-wrap; overflow-wrap: anywhere; max-height: 400px; overflow: auto; font-size: 13px; }
|
||||
@media (max-width: 480px) { .analytics-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } .metric { padding: 12px; } .metric b { font-size: 22px; } }
|
||||
.deep-grid { display: grid; grid-template-columns: repeat(auto-fit,minmax(min(100%,240px),1fr)); gap: 12px; }
|
||||
.deep-grid label { display: grid; gap: 5px; min-width: 0; color: var(--text); font-size: 13px; }
|
||||
.deep-search .muted { color: var(--text-2); }
|
||||
.deep-grid .field { width: 100%; flex: initial; }
|
||||
.deep-grid textarea.field { height: auto; min-height: 110px; padding: 10px 12px; resize: vertical; }
|
||||
.deep-job { padding: 14px 0; border-bottom: 1px solid var(--line-2); overflow-wrap: anywhere; }
|
||||
.deep-job progress { width: 100%; height: 8px; accent-color: var(--a1); }
|
||||
.deep-log { max-height: 240px; overflow: auto; white-space: pre-wrap; font: 12px/1.5 var(--mono); color: var(--text-2); }
|
||||
@media (min-width: 761px) and (max-width: 1100px) {
|
||||
.top { flex-wrap: wrap; }
|
||||
.tabs { order: 2; width: 100%; margin-left: 0; overflow-x: auto; }
|
||||
@@ -547,6 +555,29 @@
|
||||
<div class="card"><h2>Most played</h2><div id="analyticsTop"></div></div>
|
||||
</section>
|
||||
|
||||
<section id="p-deep-search" class="panel deep-search">
|
||||
<h1 class="page">Deep search</h1>
|
||||
<p class="lead">Build the local video index from YouTube searches. Runs are owned by the server and continue if this page closes.</p>
|
||||
<div class="card"><h2>Query plan</h2><p class="muted">One seed per line. Presets include worship terms and artists in English, Tagalog, Cebuano and Spanish.</p>
|
||||
<form id="deepSearchForm">
|
||||
<div class="deep-grid">
|
||||
<label>Seed terms<textarea class="field" id="deepSeeds" required></textarea></label>
|
||||
<label>Modifiers, one per line<textarea class="field" id="deepModifiers"></textarea></label>
|
||||
<label>Maximum queries<input class="field" id="deepMaxQueries" type="number" min="1" max="500" value="160" /></label>
|
||||
<label>Results per query<input class="field" id="deepResults" type="number" min="1" max="200" value="50" /></label>
|
||||
<label>Related videos<select class="field" id="deepDepth"><option value="0">Off</option><option value="1">Top hits · depth 1</option><option value="2">Top hits · depth 2</option></select></label>
|
||||
<label>Related results per query<input class="field" id="deepRelatedTop" type="number" min="1" max="10" value="3" /></label>
|
||||
<label>Channel IDs or URLs<textarea class="field" id="deepChannels" placeholder="UC… or https://www.youtube.com/channel/UC…"></textarea></label>
|
||||
<label>Playlist IDs or URLs<textarea class="field" id="deepPlaylists" placeholder="PL… or https://www.youtube.com/playlist?list=PL…"></textarea></label>
|
||||
<label>Politeness delay (seconds)<input class="field" id="deepDelay" type="number" min="1" max="300" value="10" /></label>
|
||||
<label>Daily query cap<input class="field" id="deepDailyCap" type="number" min="1" max="10000" value="1000" /></label>
|
||||
</div>
|
||||
<div class="row" style="margin-top:12px"><button class="btn" type="button" id="deepEstimate">Estimate plan</button><button class="btn pri" type="submit" id="deepStart">Start deep search</button><span class="muted" id="deepEstimateText" role="status" aria-live="polite"></span></div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="card"><div class="row"><h2 style="margin:0">Runs</h2><span class="grow"></span><button class="btn sm" type="button" id="deepRefresh">Refresh</button></div><p class="muted" id="deepIndexStatus"></p><div id="deepJobs"></div></div>
|
||||
</section>
|
||||
|
||||
<section id="p-access" class="panel">
|
||||
<h1 class="page">API access</h1>
|
||||
<p class="lead">Tokens let scripts write shared lyrics and chapters. A token is shown once — store it in your secrets vault.</p>
|
||||
@@ -608,9 +639,10 @@
|
||||
peers: '<circle cx="6" cy="12" r="2.5"/><circle cx="18" cy="6" r="2.5"/><circle cx="18" cy="18" r="2.5"/><path d="M8.3 10.9l7.4-3.8M8.3 13.1l7.4 3.8"/>',
|
||||
history: '<path d="M3 12a9 9 0 109-9 9 9 0 00-6.4 2.6L3 8"/><path d="M3 3v5h5M12 7v5l3 2"/>',
|
||||
analytics: '<path d="M4 20h16M7 16V9M12 16V4M17 16v-5"/>',
|
||||
'deep-search': '<circle cx="10.5" cy="10.5" r="6.5"/><path d="m15.5 15.5 5 5M8 10.5h5M10.5 8v5"/>',
|
||||
access: '<circle cx="8" cy="15" r="4"/><path d="M11 12l9-9M16 7l3 3M14 9l2 2"/>',
|
||||
};
|
||||
const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['access', 'API']];
|
||||
const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['access', 'API']];
|
||||
let tab = 'editor';
|
||||
function buildTabs() {
|
||||
const html = TABS.map(([k, l]) => `<button class="tab" role="tab" data-tab="${k}" aria-selected="false"><svg viewBox="0 0 24 24">${ICONS[k]}</svg><span>${l}</span>${k === 'reports' ? '<span class="badge hidden" data-badge>0</span>' : ''}</button>`).join('');
|
||||
@@ -630,6 +662,7 @@
|
||||
if (name === 'library') loadUploads();
|
||||
if (name === 'access') loadTokens();
|
||||
if (name === 'analytics') loadAnalytics();
|
||||
if (name === 'deep-search') loadDeepSearch();
|
||||
window.scrollTo({ top: 0 });
|
||||
}
|
||||
function setBadge(n) {
|
||||
@@ -723,6 +756,52 @@
|
||||
$('metadataFilterForm').addEventListener('submit', e => { e.preventDefault(); metadataOffset = 0; loadMetadata(); });
|
||||
$('metadataPrev').addEventListener('click', () => { metadataOffset = Math.max(0, metadataOffset - 50); loadMetadata(); });
|
||||
$('metadataNext').addEventListener('click', () => { metadataOffset += 50; loadMetadata(); });
|
||||
|
||||
/* ---------------- durable deep search ---------------- */
|
||||
let deepTimer = null;
|
||||
const lines = id => $(id).value.split(/\r?\n/).map(s => s.trim()).filter(Boolean);
|
||||
function deepBody() {
|
||||
return { seeds: lines('deepSeeds'), modifiers: lines('deepModifiers'), maxQueries: Number($('deepMaxQueries').value),
|
||||
resultsPerQuery: Number($('deepResults').value), relatedDepth: Number($('deepDepth').value), relatedTop: Number($('deepRelatedTop').value),
|
||||
channels: lines('deepChannels'), playlists: lines('deepPlaylists'), delaySeconds: Number($('deepDelay').value), dailyCap: Number($('deepDailyCap').value) };
|
||||
}
|
||||
if (!$('deepSeeds').value) $('deepSeeds').value = 'worship\npraise\nhymns\ngospel\npagsamba\npapuri\nmga awit ng papuri\nebanghelyo\ndayeg\npagdayeg\nmga alawiton\nmaayong balita\nadoración\nalabanza\nhimnos\nevangelio\nHillsong Worship\nElevation Worship\nBethel Music\nPlanetshakers\nDon Moen\nSandi Patty';
|
||||
if (!$('deepModifiers').value) $('deepModifiers').value = 'lyrics\nlive\nacoustic\ninstrumental\nkaraoke';
|
||||
$('deepEstimate').addEventListener('click', async () => {
|
||||
const label = $('deepEstimateText'); label.textContent = 'Counting the plan…';
|
||||
try { const j = await api('/api/admin/deep-search/estimate', { method: 'POST', body: deepBody() });
|
||||
label.textContent = j.ok ? `${j.queries} unique queries · ${j.browseSources} browse sources · about ${j.estimatedMinutes} min · ${j.indexSize.toLocaleString()} indexed · ${j.dailyRemaining} daily requests remain` : j.error || 'Estimate unavailable';
|
||||
} catch { label.textContent = 'Could not reach the server.'; }
|
||||
});
|
||||
$('deepSearchForm').addEventListener('submit', async e => {
|
||||
e.preventDefault(); const button = $('deepStart'); button.disabled = true;
|
||||
try { const j = await api('/api/admin/deep-search/start', { method: 'POST', body: deepBody() });
|
||||
$('deepEstimateText').textContent = j.ok ? 'Job queued. It will continue if this page closes.' : j.error || 'Could not start job.';
|
||||
await loadDeepSearch();
|
||||
} catch { $('deepEstimateText').textContent = 'Could not reach the server.'; }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
async function loadDeepSearch() {
|
||||
try {
|
||||
const j = await api('/api/admin/deep-search/jobs'); if (!j.ok) throw Error(j.error || 'Could not load deep-search jobs.');
|
||||
$('deepIndexStatus').textContent = `${Number(j.indexSize).toLocaleString()} videos in the local search index.`;
|
||||
$('deepJobs').innerHTML = j.jobs.length ? j.jobs.map(job => {
|
||||
const active = ['queued','running','paused'].includes(job.status), progress = job.queriesTotal ? Math.round(job.queriesDone / job.queriesTotal * 100) : 0;
|
||||
const eta = job.etaMs == null ? 'estimating' : job.etaMs < 60000 ? `${Math.ceil(job.etaMs / 1000)} sec` : `${Math.ceil(job.etaMs / 60000)} min`;
|
||||
const controls = job.status === 'running' || job.status === 'queued' ? `<button class="btn sm" data-deep-action="pause" data-id="${esc(job.id)}">Pause</button>` : job.status === 'paused' ? `<button class="btn sm" data-deep-action="resume" data-id="${esc(job.id)}">Resume</button>` : '';
|
||||
return `<article class="deep-job"><div class="row"><b>${esc(job.status[0].toUpperCase() + job.status.slice(1))}</b><span class="muted">${job.queriesDone}/${job.queriesTotal} queries · ${job.newVideos} new videos · ETA ${eta}</span><span class="grow"></span>${controls}${active ? `<button class="btn sm dng" data-deep-action="cancel" data-id="${esc(job.id)}">Cancel</button>` : ''}</div><progress max="100" value="${progress}" aria-label="Deep search progress"></progress>${job.current ? `<p class="muted">Current: ${esc(job.current)}</p>` : ''}${job.errors?.length ? `<details><summary>${job.errors.length} errors</summary>${job.errors.map(x => `<p>${esc(x.query)}: ${esc(x.error)}</p>`).join('')}</details>` : ''}<details><summary>Live log (${job.log?.length || 0})</summary><div class="deep-log">${(job.log || []).map(x => `${new Date(x.at).toLocaleTimeString()} ${esc(x.message)}`).join('\n')}</div></details></article>`;
|
||||
}).join('') : '<p class="muted">No deep-search runs yet.</p>';
|
||||
clearTimeout(deepTimer); deepTimer = null;
|
||||
if (tab === 'deep-search' && j.jobs.some(job => ['queued','running'].includes(job.status))) deepTimer = setTimeout(loadDeepSearch, 3000);
|
||||
} catch (error) { $('deepIndexStatus').textContent = error.message || 'Could not load deep-search progress.'; }
|
||||
}
|
||||
$('deepRefresh').addEventListener('click', loadDeepSearch);
|
||||
$('deepJobs').addEventListener('click', async e => {
|
||||
const button = e.target.closest('[data-deep-action]'); if (!button) return;
|
||||
button.disabled = true;
|
||||
try { const j = await api(`/api/admin/deep-search/${encodeURIComponent(button.dataset.id)}/${button.dataset.deepAction}`, { method: 'POST', body: {} }); if (!j.ok) $('deepIndexStatus').textContent = j.error || `Could not ${button.dataset.deepAction} job.`; }
|
||||
finally { await loadDeepSearch(); }
|
||||
});
|
||||
$('metadataTable').addEventListener('click', async e => {
|
||||
const btn = e.target.closest('[data-metadata-id]'); if (!btn) return;
|
||||
const j = await api('/api/admin/metadata/' + btn.dataset.metadataId); if (!j.ok) return;
|
||||
@@ -902,7 +981,7 @@
|
||||
const [m, u] = await Promise.all([api('/api/admin/media'), api('/api/uploads?limit=200')]);
|
||||
const seen = new Set();
|
||||
SONGS = [];
|
||||
for (const r of (m.media || [])) { seen.add(r.id); SONGS.push({ id: r.id, title: r.title || r.id, sub: r.channel, lines: r.lyricsLines || 0 }); }
|
||||
for (const r of (m.media || [])) { seen.add(r.id); SONGS.push({ id: r.id, title: r.title || r.id, sub: r.channel, lines: r.lyricsLines || 0, sourceUrl: r.sourceUrl }); }
|
||||
for (const r of (u.uploads || [])) if (!seen.has(r.id)) SONGS.push({ id: r.id, title: r.title, sub: r.artist || 'Upload', lines: -1 });
|
||||
paintSongs();
|
||||
}
|
||||
@@ -912,6 +991,7 @@
|
||||
$('songList').innerHTML = rows.length ? rows.map((r) => `<div class="song-row"><button type="button" class="song-open" data-open="${esc(r.id)}">
|
||||
<span style="min-width:0;flex:1"><b>${esc(r.title)}</b><small>${esc(r.sub || '')}</small></span>
|
||||
<span class="st ${r.lines > 0 ? 'has' : ''}">${r.lines > 0 ? `${r.lines} lines` : r.lines === 0 ? 'no lyrics' : 'open'}</span></button>
|
||||
${r.sourceUrl ? `<a class="btn sm" href="${esc(r.sourceUrl)}" target="_blank" rel="noopener">Source</a><button type="button" class="btn sm" data-redownload="${esc(r.id)}">Re-download from source</button>` : ''}
|
||||
<button type="button" class="btn sm whisper-request" data-whisper="${esc(r.id)}" aria-label="Transcribe ${esc(r.title)} with Whisper">Whisper</button></div>`).join('')
|
||||
: '<div class="muted" style="padding:10px">Nothing matches — paste a link above to open any YouTube song.</div>';
|
||||
paintWhisper();
|
||||
@@ -919,8 +999,17 @@
|
||||
$('songFilter').addEventListener('input', paintSongs);
|
||||
$('songList').addEventListener('click', (e) => {
|
||||
const whisper = e.target.closest('[data-whisper]'); if (whisper) { startWhisper(whisper.dataset.whisper); return; }
|
||||
const source = e.target.closest('[data-redownload]'); if (source) { startSourceRedownload(source); return; }
|
||||
const b = e.target.closest('[data-open]'); if (b) leOpen(b.dataset.open);
|
||||
});
|
||||
async function startSourceRedownload(button) {
|
||||
button.disabled = true; const original = button.textContent; button.textContent = 'Queueing…';
|
||||
try { const result = await api(`/api/admin/media/${encodeURIComponent(button.dataset.redownload)}/redownload`, { method: 'POST', body: {} });
|
||||
button.textContent = result.ok ? 'Queued' : 'Could not queue';
|
||||
button.title = result.error || 'The server-owned download continues if this page closes.';
|
||||
} catch { button.textContent = 'Connection failed'; }
|
||||
setTimeout(() => { button.textContent = original; button.disabled = false; }, 2000);
|
||||
}
|
||||
$('recentChips').addEventListener('click', (e) => { const b = e.target.closest('[data-open]'); if (b) leOpen(b.dataset.open); });
|
||||
|
||||
/* ---------------- p2p ---------------- */
|
||||
|
||||
3
playwright.admin-deep-search.config.js
Normal file
3
playwright.admin-deep-search.config.js
Normal file
@@ -0,0 +1,3 @@
|
||||
const { defineConfig } = require('@playwright/test');
|
||||
const classic = require('./playwright.classic.config');
|
||||
module.exports = defineConfig({ ...classic, testMatch: /admin-deep-search\.spec\.js/ });
|
||||
@@ -168,6 +168,15 @@ export async function initDb() {
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_collections_queue ON metadata_collections (status,created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS deep_search_jobs (
|
||||
id TEXT PRIMARY KEY, status TEXT NOT NULL, plan TEXT NOT NULL, options TEXT NOT NULL,
|
||||
state TEXT NOT NULL DEFAULT '{}', error TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_deep_search_jobs ON deep_search_jobs(status,created_at);
|
||||
CREATE TABLE IF NOT EXISTS deep_search_daily (
|
||||
day TEXT PRIMARY KEY, queries INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS lyric_transcriptions (
|
||||
id TEXT PRIMARY KEY, video_id TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'queued',
|
||||
base_rev INTEGER NOT NULL DEFAULT 0, result TEXT, error TEXT, stage TEXT NOT NULL DEFAULT 'queued',
|
||||
|
||||
214
server/deep-search.js
Normal file
214
server/deep-search.js
Normal file
@@ -0,0 +1,214 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { db as defaultDb } from './db.js';
|
||||
import * as catalog from './video-catalog.js';
|
||||
import * as innertube from './innertube.js';
|
||||
import * as searchCache from './search-cache.js';
|
||||
|
||||
const MAX_SEEDS = 200, MAX_PLAN = 500, MAX_ERRORS = 50;
|
||||
const clean = value => String(value || '').replace(/[\u0000-\u001f\u007f]/g, ' ').trim().slice(0, 200);
|
||||
const fold = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase().replace(/\s+/g, ' ').trim();
|
||||
const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } };
|
||||
const sleepDefault = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
export const DEFAULT_PRESETS = [
|
||||
'worship', 'praise', 'hymns', 'gospel', 'pagsamba', 'papuri', 'mga awit ng papuri', 'ebanghelyo',
|
||||
'dayeg', 'pagdayeg', 'mga alawiton', 'maayong balita', 'adoración', 'alabanza', 'himnos', 'evangelio',
|
||||
'Hillsong Worship', 'Elevation Worship', 'Bethel Music', 'Planetshakers', 'Don Moen', 'Sandi Patty',
|
||||
];
|
||||
export const DEFAULT_MODIFIERS = ['lyrics', 'live', 'acoustic', 'instrumental', 'karaoke'];
|
||||
|
||||
export function expandQueryPlan({ seeds = [], modifiers = [], maxQueries = 80 } = {}) {
|
||||
const seedList = (Array.isArray(seeds) ? seeds : String(seeds).split(/\r?\n/)).map(clean).filter(Boolean).slice(0, MAX_SEEDS);
|
||||
const mods = (Array.isArray(modifiers) ? modifiers : String(modifiers).split(/[,\r\n]+/)).map(clean).filter(Boolean).slice(0, 30);
|
||||
const cap = Math.max(1, Math.min(MAX_PLAN, Number(maxQueries) || 1));
|
||||
const out = [], seen = new Set();
|
||||
for (const seed of seedList) for (const modifier of ['', ...mods]) {
|
||||
const query = clean(modifier ? `${seed} ${modifier}` : seed), key = fold(query);
|
||||
if (key && !seen.has(key)) { seen.add(key); out.push(query); if (out.length >= cap) return out; }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function publicJob(row, indexSize = 0) {
|
||||
const state = parse(row.state);
|
||||
const done = Number(state.queriesDone) || 0, total = JSON.parse(row.plan || '[]').length;
|
||||
const elapsed = Math.max(0, Date.now() - Number(row.created_at));
|
||||
const etaMs = done ? Math.max(0, Math.round(elapsed / done * (total - done))) : null;
|
||||
return { id: row.id, status: row.status, plan: JSON.parse(row.plan || '[]'), options: parse(row.options),
|
||||
queriesDone: done, queriesTotal: total, current: state.current || '', newVideos: Number(state.newVideos) || 0,
|
||||
totalIndexSize: indexSize, errors: state.errors || [], log: state.log || [], etaMs,
|
||||
createdAt: Number(row.created_at), updatedAt: Number(row.updated_at), error: row.error || null };
|
||||
}
|
||||
|
||||
export function createDeepSearchService({ db = defaultDb, searchDeep = innertube.searchDeep, related = innertube.related,
|
||||
browseUploads = innertube.browseUploads, ingest = catalog.ingest, putSearch = searchCache.put,
|
||||
sleep = sleepDefault, random = Math.random, now = Date.now, autoStart = true } = {}) {
|
||||
let worker = null;
|
||||
let timer = null;
|
||||
const log = (state, message) => { state.log ||= []; state.log.push({ at: now(), message: String(message).slice(0, 240) }); if (state.log.length > 100) state.log.splice(0, state.log.length - 100); };
|
||||
async function indexSize() { return Number((await db.execute('SELECT COUNT(*) AS n FROM video_meta')).rows[0]?.n || 0); }
|
||||
async function getJob(id) { return (await db.execute({ sql: 'SELECT * FROM deep_search_jobs WHERE id=?', args: [id] })).rows[0] || null; }
|
||||
async function checkpoint(job, state, status = 'running', error = null) {
|
||||
const r = await db.execute({ sql: "UPDATE deep_search_jobs SET state=?,status=?,error=?,updated_at=? WHERE id=? AND status='running'", args: [JSON.stringify(state), status, error, now(), job.id] });
|
||||
if (!r.rowsAffected) { const stop = new Error('Deep search paused or cancelled.'); stop.name = 'DeepSearchStopped'; throw stop; }
|
||||
}
|
||||
async function reserveDaily(cap) {
|
||||
const day = new Date(now()).toISOString().slice(0, 10);
|
||||
const r = await db.execute({ sql: `INSERT INTO deep_search_daily(day,queries) VALUES(?,1)
|
||||
ON CONFLICT(day) DO UPDATE SET queries=queries+1 WHERE queries<? RETURNING queries`, args: [day, cap] });
|
||||
return r.rows.length ? Number(r.rows[0].queries) : null;
|
||||
}
|
||||
async function run(job) {
|
||||
const options = parse(job.options), plan = JSON.parse(job.plan || '[]');
|
||||
const state = { queriesDone: 0, newVideos: 0, errors: [], log: [], current: '', relatedSeen: [], ...parse(job.state) };
|
||||
async function saveCards(query, cards, cacheCards = cards) {
|
||||
const unique = [...new Map((cards || []).filter(c => c?.id).map(c => [c.id, c])).values()];
|
||||
if (unique.length) {
|
||||
const validIds = unique.filter(c => /^[\w-]{11}$/.test(c.id)).map(c => c.id);
|
||||
const existing = validIds.length ? (await db.execute({ sql: `SELECT id FROM video_meta WHERE id IN (${validIds.map(() => '?').join(',')})`, args: validIds })).rows : [];
|
||||
const present = new Set(existing.map(row => row.id));
|
||||
await ingest(unique, 'deep');
|
||||
state.newVideos += validIds.filter(id => !present.has(id)).length;
|
||||
}
|
||||
await putSearch(query, (cacheCards || unique).slice(0, 200));
|
||||
}
|
||||
async function requireDailyBudget() {
|
||||
const used = await reserveDaily(Math.max(1, Number(options.dailyCap) || 1000));
|
||||
if (used !== null) return;
|
||||
state.dailyPausedDay = new Date(now()).toISOString().slice(0, 10);
|
||||
log(state, 'Daily query cap reached; job will resume tomorrow.');
|
||||
await checkpoint(job, state, 'paused');
|
||||
const stop = new Error('Daily cap reached.'); stop.name = 'DeepSearchStopped'; throw stop;
|
||||
}
|
||||
try {
|
||||
for (let qi = state.queriesDone; qi < plan.length; qi++) {
|
||||
const query = plan[qi];
|
||||
if (state.current !== query) state.currentIds = [];
|
||||
state.current = query; await checkpoint(job, state);
|
||||
await requireDailyBudget();
|
||||
await checkpoint(job, state);
|
||||
let cards, lastError;
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
try {
|
||||
cards = await searchDeep(query, { limit: Math.max(1, Math.min(200, Number(options.resultsPerQuery) || 50)), maxPages: 14, timeoutMs: 30000,
|
||||
pageDelayMs: Math.max(0, Number(options.delayMs) || 0), jitterMs: Math.max(0, Number(options.jitterMs) || 0),
|
||||
onPage: async pageCards => {
|
||||
await checkpoint(job, state);
|
||||
const saved = new Set(state.currentIds || []);
|
||||
const fresh = pageCards.filter(card => !saved.has(card.id));
|
||||
await saveCards(query, fresh, pageCards);
|
||||
state.currentIds = pageCards.map(card => card.id);
|
||||
await checkpoint(job, state);
|
||||
} });
|
||||
await checkpoint(job, state);
|
||||
lastError = null; break;
|
||||
} catch (error) { if (error.name === 'DeepSearchStopped') throw error; lastError = error; if (attempt < 2) await sleep(Math.min(60000, (Number(options.delayMs) || 10000) * 2 ** attempt + Math.floor(random() * 3000))); }
|
||||
}
|
||||
if (lastError) { state.errors.push({ query, error: String(lastError.message).slice(0, 200) }); if (state.errors.length > MAX_ERRORS) state.errors.shift(); log(state, `Search failed: ${query}`); }
|
||||
else {
|
||||
const saved = new Set(state.currentIds || []);
|
||||
const remaining = (cards || []).filter(card => !saved.has(card.id));
|
||||
await saveCards(query, remaining, cards || []);
|
||||
state.currentIds = (cards || []).map(card => card.id);
|
||||
const depth = Math.max(0, Math.min(2, Number(options.relatedDepth) || 0));
|
||||
if (depth) {
|
||||
let frontier = (cards || []).slice(0, Math.max(1, Math.min(10, Number(options.relatedTop) || 3))).map(c => ({ id: c.id, depth: 1 }));
|
||||
for (let level = 1; level <= depth && frontier.length; level++) {
|
||||
const next = [];
|
||||
for (const item of frontier) {
|
||||
if (state.relatedSeen.includes(item.id)) continue;
|
||||
state.relatedSeen.push(item.id);
|
||||
try {
|
||||
await sleep(Math.max(0, Number(options.delayMs) || 10000) + Math.floor(random() * (Number(options.jitterMs) || 3000)));
|
||||
await checkpoint(job, state);
|
||||
await requireDailyBudget();
|
||||
const recs = await related(item.id); await checkpoint(job, state);
|
||||
await saveCards(`${query} related ${item.id}`, recs);
|
||||
if (level < depth) next.push(...recs.slice(0, 2).map(c => ({ id: c.id, depth: level + 1 })));
|
||||
} catch (error) { if (error.name === 'DeepSearchStopped') throw error; state.errors.push({ query: `related:${item.id}`, error: String(error.message).slice(0, 200) }); }
|
||||
}
|
||||
frontier = next;
|
||||
}
|
||||
}
|
||||
log(state, `Saved ${cards?.length || 0} results for “${query}”.`);
|
||||
}
|
||||
state.queriesDone = qi + 1; state.current = ''; state.currentIds = []; await checkpoint(job, state);
|
||||
if (qi + 1 < plan.length) await sleep(Math.max(0, Number(options.delayMs) || 10000) + Math.floor(random() * (Number(options.jitterMs) || 3000)));
|
||||
}
|
||||
for (const source of [...(options.channels || []), ...(options.playlists || [])].slice(0, 50)) {
|
||||
state.current = `browse ${source}`; await checkpoint(job, state);
|
||||
try { await requireDailyBudget(); await checkpoint(job, state); const cards = await browseUploads(source, { limit: Math.max(1, Math.min(200, Number(options.resultsPerQuery) || 50)), pageDelayMs: options.delayMs, jitterMs: options.jitterMs }); await checkpoint(job, state); await saveCards(state.current, cards); log(state, `Browsed ${cards.length} videos from ${source}.`); }
|
||||
catch (error) { if (error.name === 'DeepSearchStopped') throw error; state.errors.push({ query: state.current, error: String(error.message).slice(0, 200) }); }
|
||||
await checkpoint(job, state);
|
||||
}
|
||||
state.current = ''; log(state, 'Deep search complete.'); await checkpoint(job, state, 'complete');
|
||||
} catch (error) {
|
||||
if (error.name !== 'DeepSearchStopped') {
|
||||
state.errors.push({ query: state.current, error: String(error.message).slice(0, 200) });
|
||||
await db.execute({ sql: "UPDATE deep_search_jobs SET state=?,status='failed',error=?,updated_at=? WHERE id=? AND status='running'", args: [JSON.stringify(state), String(error.message).slice(0, 300), now(), job.id] });
|
||||
}
|
||||
}
|
||||
}
|
||||
async function runNext() {
|
||||
if (worker) return worker;
|
||||
worker = (async () => {
|
||||
// Recover jobs interrupted by process restart; the current query is safe to repeat.
|
||||
await db.execute({ sql: "UPDATE deep_search_jobs SET status='queued',updated_at=? WHERE status='running'", args: [now()] });
|
||||
const today = new Date(now()).toISOString().slice(0, 10);
|
||||
const paused = (await db.execute("SELECT id,state FROM deep_search_jobs WHERE status='paused'")).rows;
|
||||
for (const row of paused) if (parse(row.state).dailyPausedDay && parse(row.state).dailyPausedDay < today)
|
||||
await db.execute({ sql: "UPDATE deep_search_jobs SET status='queued',updated_at=? WHERE id=? AND status='paused'", args: [now(), row.id] });
|
||||
const job = (await db.execute({ sql: "SELECT * FROM deep_search_jobs WHERE status='queued' ORDER BY created_at LIMIT 1" })).rows[0];
|
||||
if (job) {
|
||||
await db.execute({ sql: "UPDATE deep_search_jobs SET status='running',updated_at=? WHERE id=? AND status='queued'", args: [now(), job.id] });
|
||||
const current = await getJob(job.id); if (current?.status === 'running') await run(current);
|
||||
}
|
||||
return !!job;
|
||||
})().finally(() => { worker = null; });
|
||||
return worker;
|
||||
}
|
||||
const kick = () => runNext().catch(error => console.error('[deep search]', error.message));
|
||||
function start() { if (!timer) { timer = setInterval(kick, 15000); timer.unref?.(); } kick(); }
|
||||
if (autoStart) start();
|
||||
async function estimate(body) {
|
||||
const plan = expandQueryPlan(body);
|
||||
const browseSources = [...(Array.isArray(body?.channels) ? body.channels : []), ...(Array.isArray(body?.playlists) ? body.playlists : [])].map(clean).filter(Boolean).slice(0, 50).length;
|
||||
const operations = plan.length + browseSources + (Number(body?.relatedDepth) > 0 ? Math.min(plan.length * (Number(body?.relatedTop) || 3) * Math.min(2, Number(body.relatedDepth)), 100) : 0);
|
||||
const today = new Date(now()).toISOString().slice(0, 10);
|
||||
const used = Number((await db.execute({ sql: 'SELECT queries FROM deep_search_daily WHERE day=?', args: [today] })).rows[0]?.queries || 0);
|
||||
return { plan, queries: plan.length, browseSources, estimatedOperations: operations,
|
||||
estimatedMinutes: Math.ceil(operations * Math.max(1, Number(body?.delaySeconds) || 10) / 60),
|
||||
indexSize: await indexSize(), dailyUsed: used, dailyRemaining: Math.max(0, (Number(body?.dailyCap) || 1000) - used) };
|
||||
}
|
||||
function register(app, adminAuth) {
|
||||
app.post('/api/admin/deep-search/estimate', adminAuth, async c => { let body = {}; try { body = await c.req.json(); } catch {} return c.json({ ok: true, ...(await estimate(body)) }); });
|
||||
app.get('/api/admin/deep-search/jobs', adminAuth, async c => c.json({ ok: true, indexSize: await indexSize(), jobs: (await db.execute('SELECT * FROM deep_search_jobs ORDER BY created_at DESC LIMIT 30')).rows.map(row => publicJob(row)) }));
|
||||
app.post('/api/admin/deep-search/start', adminAuth, async c => {
|
||||
let body; try { body = await c.req.json(); } catch { return c.json({ ok: false, error: 'Invalid JSON.' }, 400); }
|
||||
const plan = expandQueryPlan(body); if (!plan.length) return c.json({ ok: false, error: 'Add at least one seed term.' }, 400);
|
||||
const active = (await db.execute("SELECT id FROM deep_search_jobs WHERE status IN ('queued','running','paused') LIMIT 1")).rows[0];
|
||||
if (active) return c.json({ ok: false, error: 'A deep-search job is already active.' }, 409);
|
||||
const options = { resultsPerQuery: Math.max(1, Math.min(200, Number(body.resultsPerQuery) || 50)), relatedDepth: Math.max(0, Math.min(2, Number(body.relatedDepth) || 0)), relatedTop: Math.max(1, Math.min(10, Number(body.relatedTop) || 3)), delayMs: Math.max(1000, Math.min(300000, (Number(body.delaySeconds) || 10) * 1000)), jitterMs: Math.max(0, Math.min(300000, (Number(body.jitterSeconds) || 3) * 1000)), dailyCap: Math.max(1, Math.min(10000, Number(body.dailyCap) || 1000)), channels: (Array.isArray(body.channels) ? body.channels : []).map(clean).filter(Boolean).slice(0, 25), playlists: (Array.isArray(body.playlists) ? body.playlists : []).map(clean).filter(Boolean).slice(0, 25) };
|
||||
const id = randomUUID(), time = now();
|
||||
await db.execute({ sql: "INSERT INTO deep_search_jobs(id,status,plan,options,state,created_at,updated_at) VALUES(?,'queued',?,?,?,?,?)", args: [id, JSON.stringify(plan), JSON.stringify(options), JSON.stringify({ queriesDone: 0, newVideos: 0, errors: [], log: [] }), time, time] });
|
||||
if (autoStart) kick(); return c.json({ ok: true, id }, 202);
|
||||
});
|
||||
for (const action of ['pause', 'resume', 'cancel']) app.post(`/api/admin/deep-search/:id/${action}`, adminAuth, async c => {
|
||||
const id = c.req.param('id');
|
||||
const allowed = action === 'pause' ? ['queued', 'running'] : action === 'resume' ? ['paused'] : ['queued', 'running', 'paused'];
|
||||
const target = action === 'pause' ? 'paused' : action === 'resume' ? 'queued' : 'cancelled';
|
||||
const marks = allowed.map(() => '?').join(',');
|
||||
const r = await db.execute({ sql: `UPDATE deep_search_jobs SET status=?,updated_at=? WHERE id=? AND status IN (${marks})`, args: [target, now(), id, ...allowed] });
|
||||
if (action === 'resume' && r.rowsAffected && autoStart) kick();
|
||||
return c.json({ ok: !!r.rowsAffected, status: r.rowsAffected ? target : undefined }, r.rowsAffected ? 200 : 409);
|
||||
});
|
||||
return { estimate, runNext, kick, start };
|
||||
}
|
||||
return { estimate, runNext, kick, start, register };
|
||||
}
|
||||
|
||||
export function registerDeepSearchRoutes(app, options = {}) {
|
||||
const service = createDeepSearchService(options);
|
||||
service.register(app, options.adminAuth);
|
||||
return service;
|
||||
}
|
||||
122
server/deep-search.test.js
Normal file
122
server/deep-search.test.js
Normal file
@@ -0,0 +1,122 @@
|
||||
import { expect, test, beforeAll } from 'bun:test';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { Hono } from 'hono';
|
||||
|
||||
const root = mkdtempSync(join(tmpdir(), 'ytp-deep-search-'));
|
||||
process.env.DB_PATH = join(root, 'test.db');
|
||||
const { initDb, db } = await import('./db.js');
|
||||
const catalog = await import('./video-catalog.js');
|
||||
const searchCache = await import('./search-cache.js');
|
||||
const { createDeepSearchService, expandQueryPlan } = await import('./deep-search.js');
|
||||
const auth = async (_c, next) => next();
|
||||
const sample = { id: 'd33psrch001', title: 'Worship in Cebuano', channel: 'Hope Choir' };
|
||||
|
||||
beforeAll(async () => { await initDb(); });
|
||||
|
||||
function makeApp(service) {
|
||||
const app = new Hono(); service.register(app, auth); return app;
|
||||
}
|
||||
async function request(app, path, method = 'GET', body) {
|
||||
const response = await app.request(path, { method, headers: body ? { 'content-type': 'application/json' } : {}, body: body ? JSON.stringify(body) : undefined });
|
||||
return { response, json: await response.json() };
|
||||
}
|
||||
const cleanDb = async () => {
|
||||
await db.batch(['DELETE FROM deep_search_jobs', 'DELETE FROM deep_search_daily', 'DELETE FROM video_meta', 'DELETE FROM video_meta_sources', 'DELETE FROM search_cache']);
|
||||
};
|
||||
|
||||
test.serial('query plan expands modifiers, folds accents and deduplicates', () => {
|
||||
expect(expandQueryPlan({ seeds: ['Adoración', 'adoracion', ' Gospel '], modifiers: ['live', 'LIVE'], maxQueries: 30 }))
|
||||
.toEqual(['Adoración', 'Adoración live', 'Gospel', 'Gospel live']);
|
||||
expect(expandQueryPlan({ seeds: ['one', 'two'], modifiers: ['live'], maxQueries: 2 })).toHaveLength(2);
|
||||
});
|
||||
|
||||
test.serial('estimate and background job ingest to catalog, source and search cache', async () => {
|
||||
await cleanDb();
|
||||
const service = createDeepSearchService({ db, ingest: catalog.ingest, putSearch: searchCache.put,
|
||||
searchDeep: async () => [sample], sleep: async () => {}, autoStart: false });
|
||||
const app = makeApp(service);
|
||||
const estimate = await request(app, '/api/admin/deep-search/estimate', 'POST', { seeds: ['worship'], modifiers: ['live','lyrics'], maxQueries: 4 });
|
||||
expect(estimate.json.plan).toEqual(['worship', 'worship live', 'worship lyrics']);
|
||||
const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], modifiers: [], maxQueries: 1, delaySeconds: 1 });
|
||||
expect(created.response.status).toBe(202);
|
||||
await service.runNext();
|
||||
const jobs = await request(app, '/api/admin/deep-search/jobs');
|
||||
expect(jobs.json.jobs[0].status).toBe('complete');
|
||||
expect(jobs.json.jobs[0].newVideos).toBe(1);
|
||||
const row = (await db.execute({ sql: 'SELECT source FROM video_meta_sources WHERE video_id=?', args: [sample.id] })).rows[0];
|
||||
expect(row.source).toBe('deep');
|
||||
expect((await searchCache.get('worship')).results[0].id).toBe(sample.id);
|
||||
});
|
||||
|
||||
test.serial('pause during upstream call does not ingest; resume continues from the persisted query', async () => {
|
||||
await cleanDb(); let begin, release, calls = 0;
|
||||
const began = new Promise(resolve => { begin = resolve; });
|
||||
const gate = new Promise(resolve => { release = resolve; });
|
||||
const service = createDeepSearchService({ db, ingest: catalog.ingest, putSearch: searchCache.put, autoStart: false,
|
||||
sleep: async () => {}, searchDeep: async () => { calls++; if (calls === 1) { begin(); await gate; } return [sample]; } });
|
||||
const app = makeApp(service);
|
||||
const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 });
|
||||
const work = service.runNext(); await began;
|
||||
await request(app, `/api/admin/deep-search/${created.json.id}/pause`, 'POST', {});
|
||||
release(); await work;
|
||||
expect((await db.execute({ sql: 'SELECT COUNT(*) AS n FROM video_meta WHERE id=?', args: [sample.id] })).rows[0].n).toBe(0);
|
||||
let jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('paused');
|
||||
await request(app, `/api/admin/deep-search/${created.json.id}/resume`, 'POST', {});
|
||||
await service.runNext();
|
||||
jobs = await request(app, '/api/admin/deep-search/jobs');
|
||||
expect(jobs.json.jobs[0].status).toBe('complete'); expect(calls).toBe(2);
|
||||
});
|
||||
|
||||
test.serial('cancelled in-flight job does not write its late result', async () => {
|
||||
await cleanDb(); let begin, release;
|
||||
const began = new Promise(resolve => { begin = resolve; }), gate = new Promise(resolve => { release = resolve; });
|
||||
const service = createDeepSearchService({ db, autoStart: false, sleep: async () => {},
|
||||
searchDeep: async () => { begin(); await gate; return [sample]; }, ingest: catalog.ingest, putSearch: searchCache.put });
|
||||
const app = makeApp(service);
|
||||
const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 });
|
||||
const work = service.runNext(); await began;
|
||||
await request(app, `/api/admin/deep-search/${created.json.id}/cancel`, 'POST', {}); release(); await work;
|
||||
expect((await db.execute({ sql: 'SELECT COUNT(*) AS n FROM video_meta WHERE id=?', args: [sample.id] })).rows[0].n).toBe(0);
|
||||
const jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('cancelled');
|
||||
});
|
||||
|
||||
test.serial('page checkpoint survives a retry without double-counting its videos', async () => {
|
||||
await cleanDb(); let calls = 0;
|
||||
const service = createDeepSearchService({ db, autoStart: false, sleep: async () => {}, ingest: catalog.ingest, putSearch: searchCache.put,
|
||||
searchDeep: async (_query, { onPage }) => {
|
||||
calls++;
|
||||
await onPage([sample]);
|
||||
if (calls === 1) throw Error('connection reset after first page');
|
||||
return [sample];
|
||||
} });
|
||||
const app = makeApp(service);
|
||||
await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 });
|
||||
await service.runNext();
|
||||
const row = (await db.execute({ sql: 'SELECT seen FROM video_meta WHERE id=?', args: [sample.id] })).rows[0];
|
||||
expect(Number(row.seen)).toBe(1);
|
||||
});
|
||||
|
||||
test.serial('daily cap pauses and worker resumes it on the next UTC day', async () => {
|
||||
await cleanDb(); let time = Date.parse('2026-10-14T23:59:00Z');
|
||||
const service = createDeepSearchService({ db, now: () => time, autoStart: false, sleep: async () => {},
|
||||
searchDeep: async query => [{ ...sample, id: query === 'worship' ? 'd33psrch001' : 'd33psrch002' }],
|
||||
ingest: catalog.ingest, putSearch: searchCache.put });
|
||||
const app = makeApp(service);
|
||||
const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship','praise'], maxQueries: 2, delaySeconds: 1, dailyCap: 1 });
|
||||
await service.runNext();
|
||||
let jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('paused');
|
||||
time += 120000; await service.runNext(); jobs = await request(app, '/api/admin/deep-search/jobs');
|
||||
expect(jobs.json.jobs[0].status).toBe('complete'); expect(jobs.json.jobs[0].queriesDone).toBe(2);
|
||||
});
|
||||
|
||||
test.serial('only one queued/running job is accepted', async () => {
|
||||
await cleanDb();
|
||||
const service = createDeepSearchService({ db, autoStart: false }); const app = makeApp(service);
|
||||
const body = { seeds: ['worship'], maxQueries: 1 };
|
||||
expect((await request(app, '/api/admin/deep-search/start', 'POST', body)).response.status).toBe(202);
|
||||
expect((await request(app, '/api/admin/deep-search/start', 'POST', body)).response.status).toBe(409);
|
||||
});
|
||||
|
||||
test.serial('cleanup test database', () => { rmSync(root, { recursive: true, force: true }); });
|
||||
@@ -79,7 +79,7 @@ export async function search(q, { fetchImpl = fetch, timeoutMs = 6000 } = {}) {
|
||||
// are a chain, so they are fetched one after another; every page is optional —
|
||||
// a failure part-way returns what was gathered. `onPage(cards)` lets a caller
|
||||
// save progress.
|
||||
export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs = 6000, maxPages = 14, onPage } = {}) {
|
||||
export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs = 6000, maxPages = 14, onPage, pageDelayMs = 0, jitterMs = 0 } = {}) {
|
||||
const post = async (body) => {
|
||||
const res = await fetchImpl('https://www.youtube.com/youtubei/v1/search?prettyPrint=false', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
@@ -93,19 +93,92 @@ export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs
|
||||
const all = [];
|
||||
const add = (cards) => { for (const c of cards) if (!seen.has(c.id)) { seen.add(c.id); all.push(c); } };
|
||||
add(first.cards);
|
||||
if (onPage) onPage(all.slice());
|
||||
if (onPage) await onPage(all.slice());
|
||||
let next = first.next;
|
||||
for (let i = 0; i < maxPages && next && all.length < limit; i++) {
|
||||
if (pageDelayMs > 0 || jitterMs > 0) await new Promise(resolve => setTimeout(resolve, Math.max(0, pageDelayMs + Math.random() * jitterMs)));
|
||||
let page;
|
||||
try { page = parseContinuation(await post({ continuation: next })); } catch { break; }
|
||||
if (!page.cards.length) break;
|
||||
add(page.cards);
|
||||
if (onPage) onPage(all.slice());
|
||||
if (onPage) await onPage(all.slice());
|
||||
next = page.next;
|
||||
}
|
||||
return all.slice(0, limit);
|
||||
}
|
||||
|
||||
function browseToken(value) {
|
||||
if (!value || typeof value !== 'object') return '';
|
||||
const token = value.continuationItemRenderer?.continuationEndpoint?.continuationCommand?.token;
|
||||
if (token) return token;
|
||||
for (const child of Object.values(value)) {
|
||||
if (Array.isArray(child)) { for (const item of child) { const found = browseToken(item); if (found) return found; } }
|
||||
else { const found = browseToken(child); if (found) return found; }
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
export function parseBrowseVideos(json) {
|
||||
const cards = new Map();
|
||||
function visit(value) {
|
||||
if (!value || typeof value !== 'object') return;
|
||||
const video = value.videoRenderer || value.gridVideoRenderer || value.richItemRenderer?.content?.videoRenderer;
|
||||
const id = video?.videoId;
|
||||
const title = video?.title?.runs?.map(run => run.text).join('') || video?.title?.simpleText || '';
|
||||
if (id && /^[\w-]{11}$/.test(id) && title) {
|
||||
const owner = video.ownerText?.runs?.[0] || video.shortBylineText?.runs?.[0] || {};
|
||||
const browse = owner.navigationEndpoint?.browseEndpoint || {};
|
||||
cards.set(id, { id, url: `https://www.youtube.com/watch?v=${id}`, title, channel: owner.text || '',
|
||||
channelId: browse.browseId || '', channelUrl: browse.canonicalBaseUrl ? `https://www.youtube.com${browse.canonicalBaseUrl}` : '',
|
||||
duration: lengthToSeconds(video.lengthText?.simpleText), thumbnail: `https://i.ytimg.com/vi/${id}/mqdefault.jpg` });
|
||||
return;
|
||||
}
|
||||
for (const child of Object.values(value)) if (Array.isArray(child)) child.forEach(visit); else visit(child);
|
||||
}
|
||||
visit(json);
|
||||
return [...cards.values()];
|
||||
}
|
||||
|
||||
// YouTube browse continuation for channel uploads and playlists. Channel IDs
|
||||
// are translated to their uploads playlist (UU…), while playlist URLs use VL.
|
||||
export async function browseUploads(source, { limit = 200, fetchImpl = fetch, timeoutMs = 8000, maxPages = 12, pageDelayMs = 0, jitterMs = 0 } = {}) {
|
||||
let id = String(source || '').trim();
|
||||
let handle = '';
|
||||
try {
|
||||
const url = new URL(id);
|
||||
id = url.searchParams.get('list') || url.pathname.match(/\/channel\/([^/]+)/)?.[1] || id;
|
||||
handle = url.pathname.match(/\/@([^/]+)/)?.[1] || '';
|
||||
} catch { /* raw channel or playlist id */ }
|
||||
if (handle) {
|
||||
const response = await fetchImpl(`https://www.youtube.com/@${encodeURIComponent(handle)}`, { signal: AbortSignal.timeout(timeoutMs) });
|
||||
if (!response.ok) throw new Error(`channel lookup: HTTP ${response.status}`);
|
||||
const html = await response.text();
|
||||
id = html.match(/"browseId"\s*:\s*"(UC[\w-]{22})"/)?.[1] || '';
|
||||
}
|
||||
let browseId;
|
||||
if (/^UC[\w-]{22}$/.test(id)) browseId = `UU${id.slice(2)}`;
|
||||
else if (/^(?:PL|UU|LL|FL|OL)[\w-]{10,}$/.test(id)) browseId = id.startsWith('UU') || id.startsWith('LL') || id.startsWith('FL') || id.startsWith('OL') ? id : `VL${id}`;
|
||||
else throw new Error('Expected a YouTube channel ID or playlist ID/URL.');
|
||||
const post = async body => {
|
||||
const response = await fetchImpl('https://www.youtube.com/youtubei/v1/browse?prettyPrint=false', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ context: { client: CLIENT }, ...body }), signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
if (!response.ok) throw new Error(`innertube browse: HTTP ${response.status}`);
|
||||
return response.json();
|
||||
};
|
||||
let page = await post({ browseId });
|
||||
const seen = new Set(), cards = [];
|
||||
for (let i = 0; i < maxPages && cards.length < limit; i++) {
|
||||
for (const card of parseBrowseVideos(page)) if (!seen.has(card.id)) { seen.add(card.id); cards.push(card); }
|
||||
const token = browseToken(page);
|
||||
if (!token || cards.length >= limit) break;
|
||||
if (pageDelayMs > 0 || jitterMs > 0) await new Promise(resolve => setTimeout(resolve, Math.max(0, pageDelayMs + Math.random() * jitterMs)));
|
||||
page = await post({ continuation: token });
|
||||
}
|
||||
return cards.slice(0, limit);
|
||||
}
|
||||
|
||||
// Watch-next recommendations use compact renderers or the newer lockup model.
|
||||
// Traverse only the recommendation contents, never player configuration/ads.
|
||||
export function parseRelated(json) {
|
||||
|
||||
@@ -7,7 +7,7 @@ test('parses video renderers into slim cards', () => {
|
||||
const r = parseSearch(fixture);
|
||||
expect(r.length).toBe(4);
|
||||
expect(r[0]).toEqual({
|
||||
id: 'nQWFzMvCfLE', title: 'What A Beautiful Name - Hillsong Worship', channel: 'Hillsong Worship',
|
||||
id: 'nQWFzMvCfLE', url: 'https://www.youtube.com/watch?v=nQWFzMvCfLE', title: 'What A Beautiful Name - Hillsong Worship', channel: 'Hillsong Worship',
|
||||
channelId: 'UC4q12NoPNySbVqwpw4iO5Vg', channelUrl: 'https://www.youtube.com/channel/UC4q12NoPNySbVqwpw4iO5Vg',
|
||||
duration: 343, thumbnail: 'https://i.ytimg.com/vi/nQWFzMvCfLE/mqdefault.jpg',
|
||||
});
|
||||
@@ -60,3 +60,18 @@ test('related HTTP failures and unsupported shapes throw for search fallback', a
|
||||
expect(() => parseRelated({ error: {} })).toThrow('response shape');
|
||||
await expect(related('aaaaaaaaaaa', { fetchImpl: async () => ({ ok: false, status: 429 }) })).rejects.toThrow('429');
|
||||
});
|
||||
|
||||
test('channel and playlist browse parses uploads and resolves handle URLs', async () => {
|
||||
const { browseUploads, parseBrowseVideos } = await import('./innertube.js');
|
||||
const renderer = { videoRenderer: { videoId: 'abcdefghijk', title: { simpleText: 'Upload' }, ownerText: { runs: [{ text: 'Choir', navigationEndpoint: { browseEndpoint: { browseId: 'UC1234567890123456789012', canonicalBaseUrl: '/@choir' } } }] }, lengthText: { simpleText: '2:10' } } };
|
||||
expect(parseBrowseVideos({ contents: [{ gridVideoRenderer: { videoId: 'abcdefghijk', title: { simpleText: 'Upload' } } }] })[0].title).toBe('Upload');
|
||||
const calls = [];
|
||||
const fetchImpl = async (url, init = {}) => {
|
||||
calls.push([String(url), init.body && JSON.parse(init.body)]);
|
||||
if (String(url).includes('/@choir')) return new Response('"browseId":"UC1234567890123456789012"');
|
||||
return Response.json({ contents: [{ gridVideoRenderer: renderer.videoRenderer }] });
|
||||
};
|
||||
const result = await browseUploads('https://www.youtube.com/@choir', { fetchImpl, limit: 5 });
|
||||
expect(result[0].id).toBe('abcdefghijk');
|
||||
expect(calls[1][1].browseId).toBe('UU1234567890123456789012');
|
||||
});
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"scripts": {
|
||||
"start": "bun server.js",
|
||||
"dev": "bun --hot server.js",
|
||||
"test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js"
|
||||
"test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.14.0",
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import { test, expect } from 'bun:test';
|
||||
import * as sc from './search-cache.js';
|
||||
import { initDb } from './db.js';
|
||||
|
||||
import { test, expect, afterAll } from 'bun:test';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
const root = mkdtempSync(join(tmpdir(), 'ytp-search-cache-'));
|
||||
process.env.DB_PATH = join(root, 'test.db');
|
||||
const sc = await import('./search-cache.js');
|
||||
const { initDb } = await import('./db.js');
|
||||
await initDb();
|
||||
afterAll(() => rmSync(root, { recursive: true, force: true }));
|
||||
const cards = (n) => Array.from({ length: n }, (_, i) => ({ id: 'v' + i, title: 't' + i }));
|
||||
|
||||
test('put/get round-trips, case-insensitive, capped at 200', async () => {
|
||||
|
||||
@@ -68,6 +68,7 @@ import { registerCatalogRoutes } from './recommendations.js';
|
||||
import QRCode from 'qrcode';
|
||||
import { createYtdlpPool } from './ytdlp-pool.js';
|
||||
import { createWarmQueue } from './warm-queue.js';
|
||||
import { registerDeepSearchRoutes } from './deep-search.js';
|
||||
import { dirname, join as pathJoin } from 'node:path';
|
||||
|
||||
// A media proxy must not die because one client's stream hit an edge case
|
||||
@@ -1113,10 +1114,11 @@ async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal, sourceUrl
|
||||
const tmp = `${tmpdir()}/${tmpBase}.mp4`;
|
||||
const ctl = new AbortController();
|
||||
entry = { waiters: 0, tmpBase, ctl };
|
||||
entry.promise = withSaveSlot(() => {
|
||||
entry.promise = withSaveSlot(async () => {
|
||||
if (ctl.signal.aborted) throw new Error('save cancelled');
|
||||
const url = sourceUrl || await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`;
|
||||
return runYtdlpResilient([
|
||||
sourceUrl || await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`,
|
||||
url,
|
||||
'--no-warnings', '--no-playlist',
|
||||
...formatArgs,
|
||||
'--limit-rate', DOWNLOAD_RATE,
|
||||
@@ -1186,14 +1188,16 @@ async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal, cachedSrc
|
||||
try {
|
||||
// 1) Grab the full source (video+audio merged) so ffmpeg has both streams
|
||||
// — unless the server cache already holds it.
|
||||
if (!cachedSrc) await withSaveSlot(() => runYtdlpResilient([
|
||||
await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`,
|
||||
'--no-warnings', '--no-playlist',
|
||||
if (!cachedSrc) await withSaveSlot(async () => {
|
||||
const url = await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`;
|
||||
return runYtdlpResilient([
|
||||
url, '--no-warnings', '--no-playlist',
|
||||
'-f', SAVE_MUX_FORMAT,
|
||||
'--merge-output-format', 'mp4',
|
||||
'--limit-rate', DOWNLOAD_RATE,
|
||||
'-o', srcTmp,
|
||||
], { signal }));
|
||||
], { signal });
|
||||
});
|
||||
// 2) Trim + concat the keep segments into the final custom video.
|
||||
await runFfmpeg([
|
||||
'-y', '-hide_banner', '-loglevel', 'error',
|
||||
@@ -2295,6 +2299,8 @@ app.get('/api/remote/qr/:code', async (c) => {
|
||||
// ============================================================================
|
||||
const notes = registerNoteRoutes(app, {
|
||||
db: notesDb,
|
||||
catalog: videoCatalog,
|
||||
media,
|
||||
getProfile,
|
||||
profileNameRe: PROFILE_NAME_RE,
|
||||
runYtdlp: runYtdlpResilient,
|
||||
@@ -2308,6 +2314,9 @@ const notes = registerNoteRoutes(app, {
|
||||
{ label: 'Database', path: dirname(process.env.DB_PATH || './data/ytplayer.db') },
|
||||
] },
|
||||
});
|
||||
const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, autoStart: false,
|
||||
searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads,
|
||||
ingest: collectVideoMetadata, putSearch: searchCacheDb.put });
|
||||
|
||||
// ============================================================================
|
||||
// Uploads — the server's own searchable media library (see uploads.js)
|
||||
@@ -2519,6 +2528,7 @@ app.get('/*', indexHtml);
|
||||
// ============================================================================
|
||||
async function main() {
|
||||
await initDb();
|
||||
deepSearch.start();
|
||||
startThumbnails();
|
||||
await p2pDb.initP2pSchema();
|
||||
console.log(`[ytplayer] DB ready`);
|
||||
|
||||
@@ -3,7 +3,7 @@ import { db } from './db.js';
|
||||
import { mergeMissing } from './media-meta-core.js';
|
||||
|
||||
const VIDEO_ID = /^[\w-]{11}$/;
|
||||
const SOURCES = new Set(['search', 'search-cache', 'client-search', 'channel', 'streams', 'playlist', 'profile', 'sync', 'related', 'backfill', 'collector', 'device-play']);
|
||||
const SOURCES = new Set(['search', 'search-cache', 'client-search', 'channel', 'streams', 'playlist', 'profile', 'sync', 'related', 'backfill', 'collector', 'deep', 'device-play']);
|
||||
const text = (v, n = 300) => typeof v === 'string' ? v.trim().slice(0, n) : '';
|
||||
const positive = v => Number.isFinite(Number(v)) && Number(v) > 0 ? Number(v) : 0;
|
||||
const canonicalThumb = id => `https://i.ytimg.com/vi/${id}/hqdefault.jpg`;
|
||||
|
||||
38
tests/admin-deep-search.spec.js
Normal file
38
tests/admin-deep-search.spec.js
Normal file
@@ -0,0 +1,38 @@
|
||||
const { test, expect } = require('@playwright/test');
|
||||
|
||||
test('admin deep-search tab estimates, starts, controls and shows persisted progress', async ({ page }) => {
|
||||
page.on('pageerror', error => console.error('pageerror:', error.message));
|
||||
const state = { jobs: [], events: [] };
|
||||
await page.route('**/api/**', async route => {
|
||||
const req = route.request(), url = new URL(req.url());
|
||||
let body = { ok: true };
|
||||
if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true };
|
||||
else if (url.pathname === '/api/admin/media') body = { ok: true, media: [] };
|
||||
else if (url.pathname === '/api/uploads') body = { ok: true, uploads: [] };
|
||||
else if (url.pathname === '/api/admin/deep-search/estimate') body = { ok: true, plan: ['worship', 'worship live'], queries: 2, estimatedMinutes: 1, indexSize: 1240, dailyRemaining: 999 };
|
||||
else if (url.pathname === '/api/admin/deep-search/start') {
|
||||
state.events.push('start'); state.jobs = [{ id: 'deep-1', status: 'running', plan: ['worship', 'worship live'], options: {}, queriesDone: 1, queriesTotal: 2, current: 'worship live', newVideos: 24, totalIndexSize: 1264, etaMs: 12000, errors: [], log: [{ at: Date.now(), message: 'Saved 24 results' }] }];
|
||||
body = { ok: true, id: 'deep-1' };
|
||||
} else if (url.pathname === '/api/admin/deep-search/jobs') body = { ok: true, indexSize: 1264, jobs: state.jobs };
|
||||
else if (/\/api\/admin\/deep-search\/deep-1\/(pause|resume|cancel)$/.test(url.pathname)) {
|
||||
const action = url.pathname.split('/').at(-1); state.events.push(action);
|
||||
state.jobs[0].status = action === 'pause' ? 'paused' : action === 'resume' ? 'running' : 'cancelled';
|
||||
body = { ok: true, status: state.jobs[0].status };
|
||||
}
|
||||
await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) });
|
||||
});
|
||||
await page.goto('/admin.html#deep-search');
|
||||
await expect(page.locator('#deepSearchForm')).toBeVisible();
|
||||
await expect(page.locator('#deepSeeds')).toHaveValue(/worship/);
|
||||
await page.locator('#deepEstimate').click();
|
||||
await expect(page.locator('#deepEstimateText')).toContainText('2 unique queries');
|
||||
await page.locator('#deepStart').click();
|
||||
await expect(page.locator('.deep-job')).toContainText('1/2 queries');
|
||||
await page.locator('[data-deep-action="pause"]').click();
|
||||
await expect(page.locator('.deep-job')).toContainText('Paused');
|
||||
await page.locator('[data-deep-action="resume"]').click();
|
||||
await expect(page.locator('.deep-job')).toContainText('Running');
|
||||
await page.locator('[data-deep-action="cancel"]').click();
|
||||
await expect(page.locator('.deep-job')).toContainText('Cancelled');
|
||||
expect(state.events).toEqual(['start', 'pause', 'resume', 'cancel']);
|
||||
});
|
||||
Reference in New Issue
Block a user