Merge branch 'codex/deep-search' into codex/admin-ops
This commit is contained in:
Jonathan Sykes
@@ -342,6 +342,14 @@
|
||||
.collection-job p { margin: 4px 0; color: var(--text-2); }
|
||||
#metadataDetail { white-space: pre-wrap; overflow-wrap: anywhere; max-height: 400px; overflow: auto; font-size: 13px; }
|
||||
@media (max-width: 480px) { .analytics-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } .metric { padding: 12px; } .metric b { font-size: 22px; } }
|
||||
.deep-grid { display: grid; grid-template-columns: repeat(auto-fit,minmax(min(100%,240px),1fr)); gap: 12px; }
|
||||
.deep-grid label { display: grid; gap: 5px; min-width: 0; color: var(--text); font-size: 13px; }
|
||||
.deep-search .muted { color: var(--text-2); }
|
||||
.deep-grid .field { width: 100%; flex: initial; }
|
||||
.deep-grid textarea.field { height: auto; min-height: 110px; padding: 10px 12px; resize: vertical; }
|
||||
.deep-job { padding: 14px 0; border-bottom: 1px solid var(--line-2); overflow-wrap: anywhere; }
|
||||
.deep-job progress { width: 100%; height: 8px; accent-color: var(--a1); }
|
||||
.deep-log { max-height: 240px; overflow: auto; white-space: pre-wrap; font: 12px/1.5 var(--mono); color: var(--text-2); }
|
||||
@media (min-width: 761px) and (max-width: 1100px) {
|
||||
.top { flex-wrap: wrap; }
|
||||
.tabs { order: 2; width: 100%; margin-left: 0; overflow-x: auto; }
|
||||
@@ -628,6 +636,29 @@
|
||||
<div class="card"><h2>Most played</h2><div id="analyticsTop"></div></div>
|
||||
</section>
|
||||
|
||||
<section id="p-deep-search" class="panel deep-search">
|
||||
<h1 class="page">Deep search</h1>
|
||||
<p class="lead">Build the local video index from YouTube searches. Runs are owned by the server and continue if this page closes.</p>
|
||||
<div class="card"><h2>Query plan</h2><p class="muted">One seed per line. Presets include worship terms and artists in English, Tagalog, Cebuano and Spanish.</p>
|
||||
<form id="deepSearchForm">
|
||||
<div class="deep-grid">
|
||||
<label>Seed terms<textarea class="field" id="deepSeeds" required></textarea></label>
|
||||
<label>Modifiers, one per line<textarea class="field" id="deepModifiers"></textarea></label>
|
||||
<label>Maximum queries<input class="field" id="deepMaxQueries" type="number" min="1" max="500" value="160" /></label>
|
||||
<label>Results per query<input class="field" id="deepResults" type="number" min="1" max="200" value="50" /></label>
|
||||
<label>Related videos<select class="field" id="deepDepth"><option value="0">Off</option><option value="1">Top hits · depth 1</option><option value="2">Top hits · depth 2</option></select></label>
|
||||
<label>Related results per query<input class="field" id="deepRelatedTop" type="number" min="1" max="10" value="3" /></label>
|
||||
<label>Channel IDs or URLs<textarea class="field" id="deepChannels" placeholder="UC… or https://www.youtube.com/channel/UC…"></textarea></label>
|
||||
<label>Playlist IDs or URLs<textarea class="field" id="deepPlaylists" placeholder="PL… or https://www.youtube.com/playlist?list=PL…"></textarea></label>
|
||||
<label>Politeness delay (seconds)<input class="field" id="deepDelay" type="number" min="1" max="300" value="10" /></label>
|
||||
<label>Daily query cap<input class="field" id="deepDailyCap" type="number" min="1" max="10000" value="1000" /></label>
|
||||
</div>
|
||||
<div class="row" style="margin-top:12px"><button class="btn" type="button" id="deepEstimate">Estimate plan</button><button class="btn pri" type="submit" id="deepStart">Start deep search</button><span class="muted" id="deepEstimateText" role="status" aria-live="polite"></span></div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="card"><div class="row"><h2 style="margin:0">Runs</h2><span class="grow"></span><button class="btn sm" type="button" id="deepRefresh">Refresh</button></div><p class="muted" id="deepIndexStatus"></p><div id="deepJobs"></div></div>
|
||||
</section>
|
||||
|
||||
<section id="p-access" class="panel">
|
||||
<h1 class="page">API access</h1>
|
||||
<p class="lead">Tokens let scripts write shared lyrics and chapters. A token is shown once — store it in your secrets vault.</p>
|
||||
@@ -683,9 +714,10 @@
|
||||
peers: '<circle cx="6" cy="12" r="2.5"/><circle cx="18" cy="6" r="2.5"/><circle cx="18" cy="18" r="2.5"/><path d="M8.3 10.9l7.4-3.8M8.3 13.1l7.4 3.8"/>',
|
||||
history: '<path d="M3 12a9 9 0 109-9 9 9 0 00-6.4 2.6L3 8"/><path d="M3 3v5h5M12 7v5l3 2"/>',
|
||||
analytics: '<path d="M4 20h16M7 16V9M12 16V4M17 16v-5"/>',
|
||||
'deep-search': '<circle cx="10.5" cy="10.5" r="6.5"/><path d="m15.5 15.5 5 5M8 10.5h5M10.5 8v5"/>',
|
||||
access: '<circle cx="8" cy="15" r="4"/><path d="M11 12l9-9M16 7l3 3M14 9l2 2"/>',
|
||||
};
|
||||
const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['access', 'API']];
|
||||
const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['access', 'API']];
|
||||
let tab = 'editor';
|
||||
function buildTabs() {
|
||||
const html = TABS.map(([k, l]) => `<button class="tab" role="tab" data-tab="${k}" aria-selected="false"><svg viewBox="0 0 24 24">${ICONS[k]}</svg><span>${l}</span>${k === 'reports' ? '<span class="badge hidden" data-badge>0</span>' : ''}</button>`).join('');
|
||||
@@ -705,6 +737,7 @@
|
||||
if (name === 'library') loadUploads();
|
||||
if (name === 'access') loadTokens();
|
||||
if (name === 'analytics') loadAnalytics();
|
||||
if (name === 'deep-search') loadDeepSearch();
|
||||
window.scrollTo({ top: 0 });
|
||||
}
|
||||
function setBadge(n) {
|
||||
@@ -798,6 +831,52 @@
|
||||
$('metadataFilterForm').addEventListener('submit', e => { e.preventDefault(); metadataOffset = 0; loadMetadata(); });
|
||||
$('metadataPrev').addEventListener('click', () => { metadataOffset = Math.max(0, metadataOffset - 50); loadMetadata(); });
|
||||
$('metadataNext').addEventListener('click', () => { metadataOffset += 50; loadMetadata(); });
|
||||
|
||||
/* ---------------- durable deep search ---------------- */
|
||||
let deepTimer = null;
|
||||
const lines = id => $(id).value.split(/\r?\n/).map(s => s.trim()).filter(Boolean);
|
||||
function deepBody() {
|
||||
return { seeds: lines('deepSeeds'), modifiers: lines('deepModifiers'), maxQueries: Number($('deepMaxQueries').value),
|
||||
resultsPerQuery: Number($('deepResults').value), relatedDepth: Number($('deepDepth').value), relatedTop: Number($('deepRelatedTop').value),
|
||||
channels: lines('deepChannels'), playlists: lines('deepPlaylists'), delaySeconds: Number($('deepDelay').value), dailyCap: Number($('deepDailyCap').value) };
|
||||
}
|
||||
if (!$('deepSeeds').value) $('deepSeeds').value = 'worship\npraise\nhymns\ngospel\npagsamba\npapuri\nmga awit ng papuri\nebanghelyo\ndayeg\npagdayeg\nmga alawiton\nmaayong balita\nadoración\nalabanza\nhimnos\nevangelio\nHillsong Worship\nElevation Worship\nBethel Music\nPlanetshakers\nDon Moen\nSandi Patty';
|
||||
if (!$('deepModifiers').value) $('deepModifiers').value = 'lyrics\nlive\nacoustic\ninstrumental\nkaraoke';
|
||||
$('deepEstimate').addEventListener('click', async () => {
|
||||
const label = $('deepEstimateText'); label.textContent = 'Counting the plan…';
|
||||
try { const j = await api('/api/admin/deep-search/estimate', { method: 'POST', body: deepBody() });
|
||||
label.textContent = j.ok ? `${j.queries} unique queries · ${j.browseSources} browse sources · about ${j.estimatedMinutes} min · ${j.indexSize.toLocaleString()} indexed · ${j.dailyRemaining} daily requests remain` : j.error || 'Estimate unavailable';
|
||||
} catch { label.textContent = 'Could not reach the server.'; }
|
||||
});
|
||||
$('deepSearchForm').addEventListener('submit', async e => {
|
||||
e.preventDefault(); const button = $('deepStart'); button.disabled = true;
|
||||
try { const j = await api('/api/admin/deep-search/start', { method: 'POST', body: deepBody() });
|
||||
$('deepEstimateText').textContent = j.ok ? 'Job queued. It will continue if this page closes.' : j.error || 'Could not start job.';
|
||||
await loadDeepSearch();
|
||||
} catch { $('deepEstimateText').textContent = 'Could not reach the server.'; }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
async function loadDeepSearch() {
|
||||
try {
|
||||
const j = await api('/api/admin/deep-search/jobs'); if (!j.ok) throw Error(j.error || 'Could not load deep-search jobs.');
|
||||
$('deepIndexStatus').textContent = `${Number(j.indexSize).toLocaleString()} videos in the local search index.`;
|
||||
$('deepJobs').innerHTML = j.jobs.length ? j.jobs.map(job => {
|
||||
const active = ['queued','running','paused'].includes(job.status), progress = job.queriesTotal ? Math.round(job.queriesDone / job.queriesTotal * 100) : 0;
|
||||
const eta = job.etaMs == null ? 'estimating' : job.etaMs < 60000 ? `${Math.ceil(job.etaMs / 1000)} sec` : `${Math.ceil(job.etaMs / 60000)} min`;
|
||||
const controls = job.status === 'running' || job.status === 'queued' ? `<button class="btn sm" data-deep-action="pause" data-id="${esc(job.id)}">Pause</button>` : job.status === 'paused' ? `<button class="btn sm" data-deep-action="resume" data-id="${esc(job.id)}">Resume</button>` : '';
|
||||
return `<article class="deep-job"><div class="row"><b>${esc(job.status[0].toUpperCase() + job.status.slice(1))}</b><span class="muted">${job.queriesDone}/${job.queriesTotal} queries · ${job.newVideos} new videos · ETA ${eta}</span><span class="grow"></span>${controls}${active ? `<button class="btn sm dng" data-deep-action="cancel" data-id="${esc(job.id)}">Cancel</button>` : ''}</div><progress max="100" value="${progress}" aria-label="Deep search progress"></progress>${job.current ? `<p class="muted">Current: ${esc(job.current)}</p>` : ''}${job.errors?.length ? `<details><summary>${job.errors.length} errors</summary>${job.errors.map(x => `<p>${esc(x.query)}: ${esc(x.error)}</p>`).join('')}</details>` : ''}<details><summary>Live log (${job.log?.length || 0})</summary><div class="deep-log">${(job.log || []).map(x => `${new Date(x.at).toLocaleTimeString()} ${esc(x.message)}`).join('\n')}</div></details></article>`;
|
||||
}).join('') : '<p class="muted">No deep-search runs yet.</p>';
|
||||
clearTimeout(deepTimer); deepTimer = null;
|
||||
if (tab === 'deep-search' && j.jobs.some(job => ['queued','running'].includes(job.status))) deepTimer = setTimeout(loadDeepSearch, 3000);
|
||||
} catch (error) { $('deepIndexStatus').textContent = error.message || 'Could not load deep-search progress.'; }
|
||||
}
|
||||
$('deepRefresh').addEventListener('click', loadDeepSearch);
|
||||
$('deepJobs').addEventListener('click', async e => {
|
||||
const button = e.target.closest('[data-deep-action]'); if (!button) return;
|
||||
button.disabled = true;
|
||||
try { const j = await api(`/api/admin/deep-search/${encodeURIComponent(button.dataset.id)}/${button.dataset.deepAction}`, { method: 'POST', body: {} }); if (!j.ok) $('deepIndexStatus').textContent = j.error || `Could not ${button.dataset.deepAction} job.`; }
|
||||
finally { await loadDeepSearch(); }
|
||||
});
|
||||
$('metadataTable').addEventListener('click', async e => {
|
||||
const btn = e.target.closest('[data-metadata-id]'); if (!btn) return;
|
||||
const j = await api('/api/admin/metadata/' + btn.dataset.metadataId); if (!j.ok) return;
|
||||
@@ -977,7 +1056,7 @@
|
||||
const [m, u] = await Promise.all([api('/api/admin/media'), api('/api/uploads?limit=200')]);
|
||||
const seen = new Set();
|
||||
SONGS = [];
|
||||
for (const r of (m.media || [])) { seen.add(r.id); SONGS.push({ id: r.id, title: r.title || r.id, sub: r.channel, lines: r.lyricsLines || 0 }); }
|
||||
for (const r of (m.media || [])) { seen.add(r.id); SONGS.push({ id: r.id, title: r.title || r.id, sub: r.channel, lines: r.lyricsLines || 0, sourceUrl: r.sourceUrl }); }
|
||||
for (const r of (u.uploads || [])) if (!seen.has(r.id)) SONGS.push({ id: r.id, title: r.title, sub: r.artist || 'Upload', lines: -1 });
|
||||
paintSongs();
|
||||
}
|
||||
@@ -987,6 +1066,7 @@
|
||||
$('songList').innerHTML = rows.length ? rows.map((r) => `<div class="song-row"><button type="button" class="song-open" data-open="${esc(r.id)}">
|
||||
<span style="min-width:0;flex:1"><b>${esc(r.title)}</b><small>${esc(r.sub || '')}</small></span>
|
||||
<span class="st ${r.lines > 0 ? 'has' : ''}">${r.lines > 0 ? `${r.lines} lines` : r.lines === 0 ? 'no lyrics' : 'open'}</span></button>
|
||||
${r.sourceUrl ? `<a class="btn sm" href="${esc(r.sourceUrl)}" target="_blank" rel="noopener">Source</a><button type="button" class="btn sm" data-redownload="${esc(r.id)}">Re-download from source</button>` : ''}
|
||||
<button type="button" class="btn sm whisper-request" data-whisper="${esc(r.id)}" aria-label="Transcribe ${esc(r.title)} with Whisper">Whisper</button></div>`).join('')
|
||||
: '<div class="muted" style="padding:10px">Nothing matches — paste a link above to open any YouTube song.</div>';
|
||||
paintWhisper();
|
||||
@@ -994,8 +1074,17 @@
|
||||
$('songFilter').addEventListener('input', paintSongs);
|
||||
$('songList').addEventListener('click', (e) => {
|
||||
const whisper = e.target.closest('[data-whisper]'); if (whisper) { startWhisper(whisper.dataset.whisper); return; }
|
||||
const source = e.target.closest('[data-redownload]'); if (source) { startSourceRedownload(source); return; }
|
||||
const b = e.target.closest('[data-open]'); if (b) leOpen(b.dataset.open);
|
||||
});
|
||||
async function startSourceRedownload(button) {
|
||||
button.disabled = true; const original = button.textContent; button.textContent = 'Queueing…';
|
||||
try { const result = await api(`/api/admin/media/${encodeURIComponent(button.dataset.redownload)}/redownload`, { method: 'POST', body: {} });
|
||||
button.textContent = result.ok ? 'Queued' : 'Could not queue';
|
||||
button.title = result.error || 'The server-owned download continues if this page closes.';
|
||||
} catch { button.textContent = 'Connection failed'; }
|
||||
setTimeout(() => { button.textContent = original; button.disabled = false; }, 2000);
|
||||
}
|
||||
$('recentChips').addEventListener('click', (e) => { const b = e.target.closest('[data-open]'); if (b) leOpen(b.dataset.open); });
|
||||
|
||||
/* ---------------- p2p ---------------- */
|
||||
|
||||
3
playwright.admin-deep-search.config.js
Normal file
3
playwright.admin-deep-search.config.js
Normal file
@@ -0,0 +1,3 @@
|
||||
const { defineConfig } = require('@playwright/test');
|
||||
const classic = require('./playwright.classic.config');
|
||||
module.exports = defineConfig({ ...classic, testMatch: /admin-deep-search\.spec\.js/ });
|
||||
51
server/db.js
51
server/db.js
@@ -168,6 +168,15 @@ export async function initDb() {
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_collections_queue ON metadata_collections (status,created_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS deep_search_jobs (
|
||||
id TEXT PRIMARY KEY, status TEXT NOT NULL, plan TEXT NOT NULL, options TEXT NOT NULL,
|
||||
state TEXT NOT NULL DEFAULT '{}', error TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_deep_search_jobs ON deep_search_jobs(status,created_at);
|
||||
CREATE TABLE IF NOT EXISTS deep_search_daily (
|
||||
day TEXT PRIMARY KEY, queries INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS lyric_transcriptions (
|
||||
id TEXT PRIMARY KEY, video_id TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'queued',
|
||||
base_rev INTEGER NOT NULL DEFAULT 0, result TEXT, error TEXT, stage TEXT NOT NULL DEFAULT 'queued',
|
||||
@@ -257,6 +266,7 @@ export async function initDb() {
|
||||
last_used_at INTEGER
|
||||
);
|
||||
`);
|
||||
await migrateVideoMeta();
|
||||
// Uploads shared from devices (PWA share target): who sent it, and whether it
|
||||
// is listed in everyone's search (admin uploads) or reachable only by link.
|
||||
for (const col of ['owner TEXT', 'listed INTEGER NOT NULL DEFAULT 1']) {
|
||||
@@ -284,6 +294,47 @@ export async function initDb() {
|
||||
}
|
||||
}
|
||||
|
||||
const YOUTUBE_ID = /^[\w-]{11}$/;
|
||||
const folded = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
|
||||
|
||||
// These columns were added after video_meta shipped. Keep startup migration
|
||||
// idempotent and page the backfill so a large catalogue never becomes one
|
||||
// unbounded JS allocation.
|
||||
export async function migrateVideoMeta({ pageSize = 500 } = {}) {
|
||||
for (const [name, type] of [['url', 'TEXT'], ['channel_id', 'TEXT'], ['channel_url', 'TEXT'], ['published', 'TEXT'], ['lang', 'TEXT']]) {
|
||||
try { await db.execute(`ALTER TABLE video_meta ADD COLUMN ${name} ${type}`); } catch { /* column already exists */ }
|
||||
}
|
||||
await db.execute('CREATE TABLE IF NOT EXISTS catalog_schema_migrations (version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL)');
|
||||
const done = await db.execute({ sql: 'SELECT 1 FROM catalog_schema_migrations WHERE version=1' });
|
||||
if (done.rows.length) return;
|
||||
let cursor = '';
|
||||
for (;;) {
|
||||
const rows = (await db.execute({
|
||||
sql: 'SELECT id,card,hay FROM video_meta WHERE id>? ORDER BY id LIMIT ?', args: [cursor, pageSize],
|
||||
})).rows;
|
||||
if (!rows.length) break;
|
||||
const updates = [];
|
||||
for (const row of rows) {
|
||||
cursor = row.id;
|
||||
let card = {};
|
||||
try { card = JSON.parse(row.card || '{}'); } catch { /* malformed legacy card */ }
|
||||
const url = YOUTUBE_ID.test(row.id) ? `https://www.youtube.com/watch?v=${row.id}` : null;
|
||||
const channelId = card.channelId || card.channel_id || null;
|
||||
const channelUrl = card.channelUrl || card.channel_url || null;
|
||||
const published = card.published || card.publishedAt || card.upload_date || null;
|
||||
const lang = card.lang || card.language || null;
|
||||
card = { ...card, url, channelId: channelId || '', channelUrl: channelUrl || '' };
|
||||
const hay = folded([card.title, card.channel, channelId, card.description,
|
||||
...(Array.isArray(card.tags) ? card.tags : []), ...(Array.isArray(card.categories) ? card.categories : [])].filter(Boolean).join(' '));
|
||||
updates.push({ sql: 'UPDATE video_meta SET card=?,url=?,channel_id=?,channel_url=?,published=?,lang=?,hay=? WHERE id=?',
|
||||
args: [JSON.stringify(card), url, channelId, channelUrl, published, lang, hay || folded(row.hay), row.id] });
|
||||
}
|
||||
await db.batch(updates, 'write');
|
||||
if (rows.length < pageSize) break;
|
||||
}
|
||||
await db.execute({ sql: 'INSERT OR IGNORE INTO catalog_schema_migrations(version,applied_at) VALUES(1,?)', args: [Date.now()] });
|
||||
}
|
||||
|
||||
// ---- Shared video notes (lyrics / chapters) ---------------------------------
|
||||
|
||||
function noteRow(r) {
|
||||
|
||||
214
server/deep-search.js
Normal file
214
server/deep-search.js
Normal file
@@ -0,0 +1,214 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { db as defaultDb } from './db.js';
|
||||
import * as catalog from './video-catalog.js';
|
||||
import * as innertube from './innertube.js';
|
||||
import * as searchCache from './search-cache.js';
|
||||
|
||||
const MAX_SEEDS = 200, MAX_PLAN = 500, MAX_ERRORS = 50;
|
||||
const clean = value => String(value || '').replace(/[\u0000-\u001f\u007f]/g, ' ').trim().slice(0, 200);
|
||||
const fold = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase().replace(/\s+/g, ' ').trim();
|
||||
const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } };
|
||||
const sleepDefault = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
export const DEFAULT_PRESETS = [
|
||||
'worship', 'praise', 'hymns', 'gospel', 'pagsamba', 'papuri', 'mga awit ng papuri', 'ebanghelyo',
|
||||
'dayeg', 'pagdayeg', 'mga alawiton', 'maayong balita', 'adoración', 'alabanza', 'himnos', 'evangelio',
|
||||
'Hillsong Worship', 'Elevation Worship', 'Bethel Music', 'Planetshakers', 'Don Moen', 'Sandi Patty',
|
||||
];
|
||||
export const DEFAULT_MODIFIERS = ['lyrics', 'live', 'acoustic', 'instrumental', 'karaoke'];
|
||||
|
||||
export function expandQueryPlan({ seeds = [], modifiers = [], maxQueries = 80 } = {}) {
|
||||
const seedList = (Array.isArray(seeds) ? seeds : String(seeds).split(/\r?\n/)).map(clean).filter(Boolean).slice(0, MAX_SEEDS);
|
||||
const mods = (Array.isArray(modifiers) ? modifiers : String(modifiers).split(/[,\r\n]+/)).map(clean).filter(Boolean).slice(0, 30);
|
||||
const cap = Math.max(1, Math.min(MAX_PLAN, Number(maxQueries) || 1));
|
||||
const out = [], seen = new Set();
|
||||
for (const seed of seedList) for (const modifier of ['', ...mods]) {
|
||||
const query = clean(modifier ? `${seed} ${modifier}` : seed), key = fold(query);
|
||||
if (key && !seen.has(key)) { seen.add(key); out.push(query); if (out.length >= cap) return out; }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function publicJob(row, indexSize = 0) {
|
||||
const state = parse(row.state);
|
||||
const done = Number(state.queriesDone) || 0, total = JSON.parse(row.plan || '[]').length;
|
||||
const elapsed = Math.max(0, Date.now() - Number(row.created_at));
|
||||
const etaMs = done ? Math.max(0, Math.round(elapsed / done * (total - done))) : null;
|
||||
return { id: row.id, status: row.status, plan: JSON.parse(row.plan || '[]'), options: parse(row.options),
|
||||
queriesDone: done, queriesTotal: total, current: state.current || '', newVideos: Number(state.newVideos) || 0,
|
||||
totalIndexSize: indexSize, errors: state.errors || [], log: state.log || [], etaMs,
|
||||
createdAt: Number(row.created_at), updatedAt: Number(row.updated_at), error: row.error || null };
|
||||
}
|
||||
|
||||
export function createDeepSearchService({ db = defaultDb, searchDeep = innertube.searchDeep, related = innertube.related,
|
||||
browseUploads = innertube.browseUploads, ingest = catalog.ingest, putSearch = searchCache.put,
|
||||
sleep = sleepDefault, random = Math.random, now = Date.now, autoStart = true } = {}) {
|
||||
let worker = null;
|
||||
let timer = null;
|
||||
const log = (state, message) => { state.log ||= []; state.log.push({ at: now(), message: String(message).slice(0, 240) }); if (state.log.length > 100) state.log.splice(0, state.log.length - 100); };
|
||||
async function indexSize() { return Number((await db.execute('SELECT COUNT(*) AS n FROM video_meta')).rows[0]?.n || 0); }
|
||||
async function getJob(id) { return (await db.execute({ sql: 'SELECT * FROM deep_search_jobs WHERE id=?', args: [id] })).rows[0] || null; }
|
||||
async function checkpoint(job, state, status = 'running', error = null) {
|
||||
const r = await db.execute({ sql: "UPDATE deep_search_jobs SET state=?,status=?,error=?,updated_at=? WHERE id=? AND status='running'", args: [JSON.stringify(state), status, error, now(), job.id] });
|
||||
if (!r.rowsAffected) { const stop = new Error('Deep search paused or cancelled.'); stop.name = 'DeepSearchStopped'; throw stop; }
|
||||
}
|
||||
async function reserveDaily(cap) {
|
||||
const day = new Date(now()).toISOString().slice(0, 10);
|
||||
const r = await db.execute({ sql: `INSERT INTO deep_search_daily(day,queries) VALUES(?,1)
|
||||
ON CONFLICT(day) DO UPDATE SET queries=queries+1 WHERE queries<? RETURNING queries`, args: [day, cap] });
|
||||
return r.rows.length ? Number(r.rows[0].queries) : null;
|
||||
}
|
||||
async function run(job) {
|
||||
const options = parse(job.options), plan = JSON.parse(job.plan || '[]');
|
||||
const state = { queriesDone: 0, newVideos: 0, errors: [], log: [], current: '', relatedSeen: [], ...parse(job.state) };
|
||||
async function saveCards(query, cards, cacheCards = cards) {
|
||||
const unique = [...new Map((cards || []).filter(c => c?.id).map(c => [c.id, c])).values()];
|
||||
if (unique.length) {
|
||||
const validIds = unique.filter(c => /^[\w-]{11}$/.test(c.id)).map(c => c.id);
|
||||
const existing = validIds.length ? (await db.execute({ sql: `SELECT id FROM video_meta WHERE id IN (${validIds.map(() => '?').join(',')})`, args: validIds })).rows : [];
|
||||
const present = new Set(existing.map(row => row.id));
|
||||
await ingest(unique, 'deep');
|
||||
state.newVideos += validIds.filter(id => !present.has(id)).length;
|
||||
}
|
||||
await putSearch(query, (cacheCards || unique).slice(0, 200));
|
||||
}
|
||||
async function requireDailyBudget() {
|
||||
const used = await reserveDaily(Math.max(1, Number(options.dailyCap) || 1000));
|
||||
if (used !== null) return;
|
||||
state.dailyPausedDay = new Date(now()).toISOString().slice(0, 10);
|
||||
log(state, 'Daily query cap reached; job will resume tomorrow.');
|
||||
await checkpoint(job, state, 'paused');
|
||||
const stop = new Error('Daily cap reached.'); stop.name = 'DeepSearchStopped'; throw stop;
|
||||
}
|
||||
try {
|
||||
for (let qi = state.queriesDone; qi < plan.length; qi++) {
|
||||
const query = plan[qi];
|
||||
if (state.current !== query) state.currentIds = [];
|
||||
state.current = query; await checkpoint(job, state);
|
||||
await requireDailyBudget();
|
||||
await checkpoint(job, state);
|
||||
let cards, lastError;
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
try {
|
||||
cards = await searchDeep(query, { limit: Math.max(1, Math.min(200, Number(options.resultsPerQuery) || 50)), maxPages: 14, timeoutMs: 30000,
|
||||
pageDelayMs: Math.max(0, Number(options.delayMs) || 0), jitterMs: Math.max(0, Number(options.jitterMs) || 0),
|
||||
onPage: async pageCards => {
|
||||
await checkpoint(job, state);
|
||||
const saved = new Set(state.currentIds || []);
|
||||
const fresh = pageCards.filter(card => !saved.has(card.id));
|
||||
await saveCards(query, fresh, pageCards);
|
||||
state.currentIds = pageCards.map(card => card.id);
|
||||
await checkpoint(job, state);
|
||||
} });
|
||||
await checkpoint(job, state);
|
||||
lastError = null; break;
|
||||
} catch (error) { if (error.name === 'DeepSearchStopped') throw error; lastError = error; if (attempt < 2) await sleep(Math.min(60000, (Number(options.delayMs) || 10000) * 2 ** attempt + Math.floor(random() * 3000))); }
|
||||
}
|
||||
if (lastError) { state.errors.push({ query, error: String(lastError.message).slice(0, 200) }); if (state.errors.length > MAX_ERRORS) state.errors.shift(); log(state, `Search failed: ${query}`); }
|
||||
else {
|
||||
const saved = new Set(state.currentIds || []);
|
||||
const remaining = (cards || []).filter(card => !saved.has(card.id));
|
||||
await saveCards(query, remaining, cards || []);
|
||||
state.currentIds = (cards || []).map(card => card.id);
|
||||
const depth = Math.max(0, Math.min(2, Number(options.relatedDepth) || 0));
|
||||
if (depth) {
|
||||
let frontier = (cards || []).slice(0, Math.max(1, Math.min(10, Number(options.relatedTop) || 3))).map(c => ({ id: c.id, depth: 1 }));
|
||||
for (let level = 1; level <= depth && frontier.length; level++) {
|
||||
const next = [];
|
||||
for (const item of frontier) {
|
||||
if (state.relatedSeen.includes(item.id)) continue;
|
||||
state.relatedSeen.push(item.id);
|
||||
try {
|
||||
await sleep(Math.max(0, Number(options.delayMs) || 10000) + Math.floor(random() * (Number(options.jitterMs) || 3000)));
|
||||
await checkpoint(job, state);
|
||||
await requireDailyBudget();
|
||||
const recs = await related(item.id); await checkpoint(job, state);
|
||||
await saveCards(`${query} related ${item.id}`, recs);
|
||||
if (level < depth) next.push(...recs.slice(0, 2).map(c => ({ id: c.id, depth: level + 1 })));
|
||||
} catch (error) { if (error.name === 'DeepSearchStopped') throw error; state.errors.push({ query: `related:${item.id}`, error: String(error.message).slice(0, 200) }); }
|
||||
}
|
||||
frontier = next;
|
||||
}
|
||||
}
|
||||
log(state, `Saved ${cards?.length || 0} results for “${query}”.`);
|
||||
}
|
||||
state.queriesDone = qi + 1; state.current = ''; state.currentIds = []; await checkpoint(job, state);
|
||||
if (qi + 1 < plan.length) await sleep(Math.max(0, Number(options.delayMs) || 10000) + Math.floor(random() * (Number(options.jitterMs) || 3000)));
|
||||
}
|
||||
for (const source of [...(options.channels || []), ...(options.playlists || [])].slice(0, 50)) {
|
||||
state.current = `browse ${source}`; await checkpoint(job, state);
|
||||
try { await requireDailyBudget(); await checkpoint(job, state); const cards = await browseUploads(source, { limit: Math.max(1, Math.min(200, Number(options.resultsPerQuery) || 50)), pageDelayMs: options.delayMs, jitterMs: options.jitterMs }); await checkpoint(job, state); await saveCards(state.current, cards); log(state, `Browsed ${cards.length} videos from ${source}.`); }
|
||||
catch (error) { if (error.name === 'DeepSearchStopped') throw error; state.errors.push({ query: state.current, error: String(error.message).slice(0, 200) }); }
|
||||
await checkpoint(job, state);
|
||||
}
|
||||
state.current = ''; log(state, 'Deep search complete.'); await checkpoint(job, state, 'complete');
|
||||
} catch (error) {
|
||||
if (error.name !== 'DeepSearchStopped') {
|
||||
state.errors.push({ query: state.current, error: String(error.message).slice(0, 200) });
|
||||
await db.execute({ sql: "UPDATE deep_search_jobs SET state=?,status='failed',error=?,updated_at=? WHERE id=? AND status='running'", args: [JSON.stringify(state), String(error.message).slice(0, 300), now(), job.id] });
|
||||
}
|
||||
}
|
||||
}
|
||||
async function runNext() {
|
||||
if (worker) return worker;
|
||||
worker = (async () => {
|
||||
// Recover jobs interrupted by process restart; the current query is safe to repeat.
|
||||
await db.execute({ sql: "UPDATE deep_search_jobs SET status='queued',updated_at=? WHERE status='running'", args: [now()] });
|
||||
const today = new Date(now()).toISOString().slice(0, 10);
|
||||
const paused = (await db.execute("SELECT id,state FROM deep_search_jobs WHERE status='paused'")).rows;
|
||||
for (const row of paused) if (parse(row.state).dailyPausedDay && parse(row.state).dailyPausedDay < today)
|
||||
await db.execute({ sql: "UPDATE deep_search_jobs SET status='queued',updated_at=? WHERE id=? AND status='paused'", args: [now(), row.id] });
|
||||
const job = (await db.execute({ sql: "SELECT * FROM deep_search_jobs WHERE status='queued' ORDER BY created_at LIMIT 1" })).rows[0];
|
||||
if (job) {
|
||||
await db.execute({ sql: "UPDATE deep_search_jobs SET status='running',updated_at=? WHERE id=? AND status='queued'", args: [now(), job.id] });
|
||||
const current = await getJob(job.id); if (current?.status === 'running') await run(current);
|
||||
}
|
||||
return !!job;
|
||||
})().finally(() => { worker = null; });
|
||||
return worker;
|
||||
}
|
||||
const kick = () => runNext().catch(error => console.error('[deep search]', error.message));
|
||||
function start() { if (!timer) { timer = setInterval(kick, 15000); timer.unref?.(); } kick(); }
|
||||
if (autoStart) start();
|
||||
async function estimate(body) {
|
||||
const plan = expandQueryPlan(body);
|
||||
const browseSources = [...(Array.isArray(body?.channels) ? body.channels : []), ...(Array.isArray(body?.playlists) ? body.playlists : [])].map(clean).filter(Boolean).slice(0, 50).length;
|
||||
const operations = plan.length + browseSources + (Number(body?.relatedDepth) > 0 ? Math.min(plan.length * (Number(body?.relatedTop) || 3) * Math.min(2, Number(body.relatedDepth)), 100) : 0);
|
||||
const today = new Date(now()).toISOString().slice(0, 10);
|
||||
const used = Number((await db.execute({ sql: 'SELECT queries FROM deep_search_daily WHERE day=?', args: [today] })).rows[0]?.queries || 0);
|
||||
return { plan, queries: plan.length, browseSources, estimatedOperations: operations,
|
||||
estimatedMinutes: Math.ceil(operations * Math.max(1, Number(body?.delaySeconds) || 10) / 60),
|
||||
indexSize: await indexSize(), dailyUsed: used, dailyRemaining: Math.max(0, (Number(body?.dailyCap) || 1000) - used) };
|
||||
}
|
||||
function register(app, adminAuth) {
|
||||
app.post('/api/admin/deep-search/estimate', adminAuth, async c => { let body = {}; try { body = await c.req.json(); } catch {} return c.json({ ok: true, ...(await estimate(body)) }); });
|
||||
app.get('/api/admin/deep-search/jobs', adminAuth, async c => c.json({ ok: true, indexSize: await indexSize(), jobs: (await db.execute('SELECT * FROM deep_search_jobs ORDER BY created_at DESC LIMIT 30')).rows.map(row => publicJob(row)) }));
|
||||
app.post('/api/admin/deep-search/start', adminAuth, async c => {
|
||||
let body; try { body = await c.req.json(); } catch { return c.json({ ok: false, error: 'Invalid JSON.' }, 400); }
|
||||
const plan = expandQueryPlan(body); if (!plan.length) return c.json({ ok: false, error: 'Add at least one seed term.' }, 400);
|
||||
const active = (await db.execute("SELECT id FROM deep_search_jobs WHERE status IN ('queued','running','paused') LIMIT 1")).rows[0];
|
||||
if (active) return c.json({ ok: false, error: 'A deep-search job is already active.' }, 409);
|
||||
const options = { resultsPerQuery: Math.max(1, Math.min(200, Number(body.resultsPerQuery) || 50)), relatedDepth: Math.max(0, Math.min(2, Number(body.relatedDepth) || 0)), relatedTop: Math.max(1, Math.min(10, Number(body.relatedTop) || 3)), delayMs: Math.max(1000, Math.min(300000, (Number(body.delaySeconds) || 10) * 1000)), jitterMs: Math.max(0, Math.min(300000, (Number(body.jitterSeconds) || 3) * 1000)), dailyCap: Math.max(1, Math.min(10000, Number(body.dailyCap) || 1000)), channels: (Array.isArray(body.channels) ? body.channels : []).map(clean).filter(Boolean).slice(0, 25), playlists: (Array.isArray(body.playlists) ? body.playlists : []).map(clean).filter(Boolean).slice(0, 25) };
|
||||
const id = randomUUID(), time = now();
|
||||
await db.execute({ sql: "INSERT INTO deep_search_jobs(id,status,plan,options,state,created_at,updated_at) VALUES(?,'queued',?,?,?,?,?)", args: [id, JSON.stringify(plan), JSON.stringify(options), JSON.stringify({ queriesDone: 0, newVideos: 0, errors: [], log: [] }), time, time] });
|
||||
if (autoStart) kick(); return c.json({ ok: true, id }, 202);
|
||||
});
|
||||
for (const action of ['pause', 'resume', 'cancel']) app.post(`/api/admin/deep-search/:id/${action}`, adminAuth, async c => {
|
||||
const id = c.req.param('id');
|
||||
const allowed = action === 'pause' ? ['queued', 'running'] : action === 'resume' ? ['paused'] : ['queued', 'running', 'paused'];
|
||||
const target = action === 'pause' ? 'paused' : action === 'resume' ? 'queued' : 'cancelled';
|
||||
const marks = allowed.map(() => '?').join(',');
|
||||
const r = await db.execute({ sql: `UPDATE deep_search_jobs SET status=?,updated_at=? WHERE id=? AND status IN (${marks})`, args: [target, now(), id, ...allowed] });
|
||||
if (action === 'resume' && r.rowsAffected && autoStart) kick();
|
||||
return c.json({ ok: !!r.rowsAffected, status: r.rowsAffected ? target : undefined }, r.rowsAffected ? 200 : 409);
|
||||
});
|
||||
return { estimate, runNext, kick, start };
|
||||
}
|
||||
return { estimate, runNext, kick, start, register };
|
||||
}
|
||||
|
||||
export function registerDeepSearchRoutes(app, options = {}) {
|
||||
const service = createDeepSearchService(options);
|
||||
service.register(app, options.adminAuth);
|
||||
return service;
|
||||
}
|
||||
122
server/deep-search.test.js
Normal file
122
server/deep-search.test.js
Normal file
@@ -0,0 +1,122 @@
|
||||
import { expect, test, beforeAll } from 'bun:test';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { Hono } from 'hono';
|
||||
|
||||
const root = mkdtempSync(join(tmpdir(), 'ytp-deep-search-'));
|
||||
process.env.DB_PATH = join(root, 'test.db');
|
||||
const { initDb, db } = await import('./db.js');
|
||||
const catalog = await import('./video-catalog.js');
|
||||
const searchCache = await import('./search-cache.js');
|
||||
const { createDeepSearchService, expandQueryPlan } = await import('./deep-search.js');
|
||||
const auth = async (_c, next) => next();
|
||||
const sample = { id: 'd33psrch001', title: 'Worship in Cebuano', channel: 'Hope Choir' };
|
||||
|
||||
beforeAll(async () => { await initDb(); });
|
||||
|
||||
function makeApp(service) {
|
||||
const app = new Hono(); service.register(app, auth); return app;
|
||||
}
|
||||
async function request(app, path, method = 'GET', body) {
|
||||
const response = await app.request(path, { method, headers: body ? { 'content-type': 'application/json' } : {}, body: body ? JSON.stringify(body) : undefined });
|
||||
return { response, json: await response.json() };
|
||||
}
|
||||
const cleanDb = async () => {
|
||||
await db.batch(['DELETE FROM deep_search_jobs', 'DELETE FROM deep_search_daily', 'DELETE FROM video_meta', 'DELETE FROM video_meta_sources', 'DELETE FROM search_cache']);
|
||||
};
|
||||
|
||||
test.serial('query plan expands modifiers, folds accents and deduplicates', () => {
|
||||
expect(expandQueryPlan({ seeds: ['Adoración', 'adoracion', ' Gospel '], modifiers: ['live', 'LIVE'], maxQueries: 30 }))
|
||||
.toEqual(['Adoración', 'Adoración live', 'Gospel', 'Gospel live']);
|
||||
expect(expandQueryPlan({ seeds: ['one', 'two'], modifiers: ['live'], maxQueries: 2 })).toHaveLength(2);
|
||||
});
|
||||
|
||||
test.serial('estimate and background job ingest to catalog, source and search cache', async () => {
|
||||
await cleanDb();
|
||||
const service = createDeepSearchService({ db, ingest: catalog.ingest, putSearch: searchCache.put,
|
||||
searchDeep: async () => [sample], sleep: async () => {}, autoStart: false });
|
||||
const app = makeApp(service);
|
||||
const estimate = await request(app, '/api/admin/deep-search/estimate', 'POST', { seeds: ['worship'], modifiers: ['live','lyrics'], maxQueries: 4 });
|
||||
expect(estimate.json.plan).toEqual(['worship', 'worship live', 'worship lyrics']);
|
||||
const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], modifiers: [], maxQueries: 1, delaySeconds: 1 });
|
||||
expect(created.response.status).toBe(202);
|
||||
await service.runNext();
|
||||
const jobs = await request(app, '/api/admin/deep-search/jobs');
|
||||
expect(jobs.json.jobs[0].status).toBe('complete');
|
||||
expect(jobs.json.jobs[0].newVideos).toBe(1);
|
||||
const row = (await db.execute({ sql: 'SELECT source FROM video_meta_sources WHERE video_id=?', args: [sample.id] })).rows[0];
|
||||
expect(row.source).toBe('deep');
|
||||
expect((await searchCache.get('worship')).results[0].id).toBe(sample.id);
|
||||
});
|
||||
|
||||
test.serial('pause during upstream call does not ingest; resume continues from the persisted query', async () => {
|
||||
await cleanDb(); let begin, release, calls = 0;
|
||||
const began = new Promise(resolve => { begin = resolve; });
|
||||
const gate = new Promise(resolve => { release = resolve; });
|
||||
const service = createDeepSearchService({ db, ingest: catalog.ingest, putSearch: searchCache.put, autoStart: false,
|
||||
sleep: async () => {}, searchDeep: async () => { calls++; if (calls === 1) { begin(); await gate; } return [sample]; } });
|
||||
const app = makeApp(service);
|
||||
const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 });
|
||||
const work = service.runNext(); await began;
|
||||
await request(app, `/api/admin/deep-search/${created.json.id}/pause`, 'POST', {});
|
||||
release(); await work;
|
||||
expect((await db.execute({ sql: 'SELECT COUNT(*) AS n FROM video_meta WHERE id=?', args: [sample.id] })).rows[0].n).toBe(0);
|
||||
let jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('paused');
|
||||
await request(app, `/api/admin/deep-search/${created.json.id}/resume`, 'POST', {});
|
||||
await service.runNext();
|
||||
jobs = await request(app, '/api/admin/deep-search/jobs');
|
||||
expect(jobs.json.jobs[0].status).toBe('complete'); expect(calls).toBe(2);
|
||||
});
|
||||
|
||||
test.serial('cancelled in-flight job does not write its late result', async () => {
|
||||
await cleanDb(); let begin, release;
|
||||
const began = new Promise(resolve => { begin = resolve; }), gate = new Promise(resolve => { release = resolve; });
|
||||
const service = createDeepSearchService({ db, autoStart: false, sleep: async () => {},
|
||||
searchDeep: async () => { begin(); await gate; return [sample]; }, ingest: catalog.ingest, putSearch: searchCache.put });
|
||||
const app = makeApp(service);
|
||||
const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 });
|
||||
const work = service.runNext(); await began;
|
||||
await request(app, `/api/admin/deep-search/${created.json.id}/cancel`, 'POST', {}); release(); await work;
|
||||
expect((await db.execute({ sql: 'SELECT COUNT(*) AS n FROM video_meta WHERE id=?', args: [sample.id] })).rows[0].n).toBe(0);
|
||||
const jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('cancelled');
|
||||
});
|
||||
|
||||
test.serial('page checkpoint survives a retry without double-counting its videos', async () => {
|
||||
await cleanDb(); let calls = 0;
|
||||
const service = createDeepSearchService({ db, autoStart: false, sleep: async () => {}, ingest: catalog.ingest, putSearch: searchCache.put,
|
||||
searchDeep: async (_query, { onPage }) => {
|
||||
calls++;
|
||||
await onPage([sample]);
|
||||
if (calls === 1) throw Error('connection reset after first page');
|
||||
return [sample];
|
||||
} });
|
||||
const app = makeApp(service);
|
||||
await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship'], maxQueries: 1, delaySeconds: 1 });
|
||||
await service.runNext();
|
||||
const row = (await db.execute({ sql: 'SELECT seen FROM video_meta WHERE id=?', args: [sample.id] })).rows[0];
|
||||
expect(Number(row.seen)).toBe(1);
|
||||
});
|
||||
|
||||
test.serial('daily cap pauses and worker resumes it on the next UTC day', async () => {
|
||||
await cleanDb(); let time = Date.parse('2026-10-14T23:59:00Z');
|
||||
const service = createDeepSearchService({ db, now: () => time, autoStart: false, sleep: async () => {},
|
||||
searchDeep: async query => [{ ...sample, id: query === 'worship' ? 'd33psrch001' : 'd33psrch002' }],
|
||||
ingest: catalog.ingest, putSearch: searchCache.put });
|
||||
const app = makeApp(service);
|
||||
const created = await request(app, '/api/admin/deep-search/start', 'POST', { seeds: ['worship','praise'], maxQueries: 2, delaySeconds: 1, dailyCap: 1 });
|
||||
await service.runNext();
|
||||
let jobs = await request(app, '/api/admin/deep-search/jobs'); expect(jobs.json.jobs[0].status).toBe('paused');
|
||||
time += 120000; await service.runNext(); jobs = await request(app, '/api/admin/deep-search/jobs');
|
||||
expect(jobs.json.jobs[0].status).toBe('complete'); expect(jobs.json.jobs[0].queriesDone).toBe(2);
|
||||
});
|
||||
|
||||
test.serial('only one queued/running job is accepted', async () => {
|
||||
await cleanDb();
|
||||
const service = createDeepSearchService({ db, autoStart: false }); const app = makeApp(service);
|
||||
const body = { seeds: ['worship'], maxQueries: 1 };
|
||||
expect((await request(app, '/api/admin/deep-search/start', 'POST', body)).response.status).toBe(202);
|
||||
expect((await request(app, '/api/admin/deep-search/start', 'POST', body)).response.status).toBe(409);
|
||||
});
|
||||
|
||||
test.serial('cleanup test database', () => { rmSync(root, { recursive: true, force: true }); });
|
||||
@@ -20,6 +20,7 @@ function cardsFromSections(sections) {
|
||||
const path = be.canonicalBaseUrl || (be.browseId ? `/channel/${be.browseId}` : '');
|
||||
out.push({
|
||||
id: v.videoId,
|
||||
url: `https://www.youtube.com/watch?v=${v.videoId}`,
|
||||
title: v.title?.runs?.map((r) => r.text).join('') || '(untitled)',
|
||||
channel: owner.text || '',
|
||||
channelId: be.browseId || '',
|
||||
@@ -78,7 +79,7 @@ export async function search(q, { fetchImpl = fetch, timeoutMs = 6000 } = {}) {
|
||||
// are a chain, so they are fetched one after another; every page is optional —
|
||||
// a failure part-way returns what was gathered. `onPage(cards)` lets a caller
|
||||
// save progress.
|
||||
export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs = 6000, maxPages = 14, onPage } = {}) {
|
||||
export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs = 6000, maxPages = 14, onPage, pageDelayMs = 0, jitterMs = 0 } = {}) {
|
||||
const post = async (body) => {
|
||||
const res = await fetchImpl('https://www.youtube.com/youtubei/v1/search?prettyPrint=false', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
@@ -92,19 +93,92 @@ export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs
|
||||
const all = [];
|
||||
const add = (cards) => { for (const c of cards) if (!seen.has(c.id)) { seen.add(c.id); all.push(c); } };
|
||||
add(first.cards);
|
||||
if (onPage) onPage(all.slice());
|
||||
if (onPage) await onPage(all.slice());
|
||||
let next = first.next;
|
||||
for (let i = 0; i < maxPages && next && all.length < limit; i++) {
|
||||
if (pageDelayMs > 0 || jitterMs > 0) await new Promise(resolve => setTimeout(resolve, Math.max(0, pageDelayMs + Math.random() * jitterMs)));
|
||||
let page;
|
||||
try { page = parseContinuation(await post({ continuation: next })); } catch { break; }
|
||||
if (!page.cards.length) break;
|
||||
add(page.cards);
|
||||
if (onPage) onPage(all.slice());
|
||||
if (onPage) await onPage(all.slice());
|
||||
next = page.next;
|
||||
}
|
||||
return all.slice(0, limit);
|
||||
}
|
||||
|
||||
function browseToken(value) {
|
||||
if (!value || typeof value !== 'object') return '';
|
||||
const token = value.continuationItemRenderer?.continuationEndpoint?.continuationCommand?.token;
|
||||
if (token) return token;
|
||||
for (const child of Object.values(value)) {
|
||||
if (Array.isArray(child)) { for (const item of child) { const found = browseToken(item); if (found) return found; } }
|
||||
else { const found = browseToken(child); if (found) return found; }
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
export function parseBrowseVideos(json) {
|
||||
const cards = new Map();
|
||||
function visit(value) {
|
||||
if (!value || typeof value !== 'object') return;
|
||||
const video = value.videoRenderer || value.gridVideoRenderer || value.richItemRenderer?.content?.videoRenderer;
|
||||
const id = video?.videoId;
|
||||
const title = video?.title?.runs?.map(run => run.text).join('') || video?.title?.simpleText || '';
|
||||
if (id && /^[\w-]{11}$/.test(id) && title) {
|
||||
const owner = video.ownerText?.runs?.[0] || video.shortBylineText?.runs?.[0] || {};
|
||||
const browse = owner.navigationEndpoint?.browseEndpoint || {};
|
||||
cards.set(id, { id, url: `https://www.youtube.com/watch?v=${id}`, title, channel: owner.text || '',
|
||||
channelId: browse.browseId || '', channelUrl: browse.canonicalBaseUrl ? `https://www.youtube.com${browse.canonicalBaseUrl}` : '',
|
||||
duration: lengthToSeconds(video.lengthText?.simpleText), thumbnail: `https://i.ytimg.com/vi/${id}/mqdefault.jpg` });
|
||||
return;
|
||||
}
|
||||
for (const child of Object.values(value)) if (Array.isArray(child)) child.forEach(visit); else visit(child);
|
||||
}
|
||||
visit(json);
|
||||
return [...cards.values()];
|
||||
}
|
||||
|
||||
// YouTube browse continuation for channel uploads and playlists. Channel IDs
|
||||
// are translated to their uploads playlist (UU…), while playlist URLs use VL.
|
||||
export async function browseUploads(source, { limit = 200, fetchImpl = fetch, timeoutMs = 8000, maxPages = 12, pageDelayMs = 0, jitterMs = 0 } = {}) {
|
||||
let id = String(source || '').trim();
|
||||
let handle = '';
|
||||
try {
|
||||
const url = new URL(id);
|
||||
id = url.searchParams.get('list') || url.pathname.match(/\/channel\/([^/]+)/)?.[1] || id;
|
||||
handle = url.pathname.match(/\/@([^/]+)/)?.[1] || '';
|
||||
} catch { /* raw channel or playlist id */ }
|
||||
if (handle) {
|
||||
const response = await fetchImpl(`https://www.youtube.com/@${encodeURIComponent(handle)}`, { signal: AbortSignal.timeout(timeoutMs) });
|
||||
if (!response.ok) throw new Error(`channel lookup: HTTP ${response.status}`);
|
||||
const html = await response.text();
|
||||
id = html.match(/"browseId"\s*:\s*"(UC[\w-]{22})"/)?.[1] || '';
|
||||
}
|
||||
let browseId;
|
||||
if (/^UC[\w-]{22}$/.test(id)) browseId = `UU${id.slice(2)}`;
|
||||
else if (/^(?:PL|UU|LL|FL|OL)[\w-]{10,}$/.test(id)) browseId = id.startsWith('UU') || id.startsWith('LL') || id.startsWith('FL') || id.startsWith('OL') ? id : `VL${id}`;
|
||||
else throw new Error('Expected a YouTube channel ID or playlist ID/URL.');
|
||||
const post = async body => {
|
||||
const response = await fetchImpl('https://www.youtube.com/youtubei/v1/browse?prettyPrint=false', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ context: { client: CLIENT }, ...body }), signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
if (!response.ok) throw new Error(`innertube browse: HTTP ${response.status}`);
|
||||
return response.json();
|
||||
};
|
||||
let page = await post({ browseId });
|
||||
const seen = new Set(), cards = [];
|
||||
for (let i = 0; i < maxPages && cards.length < limit; i++) {
|
||||
for (const card of parseBrowseVideos(page)) if (!seen.has(card.id)) { seen.add(card.id); cards.push(card); }
|
||||
const token = browseToken(page);
|
||||
if (!token || cards.length >= limit) break;
|
||||
if (pageDelayMs > 0 || jitterMs > 0) await new Promise(resolve => setTimeout(resolve, Math.max(0, pageDelayMs + Math.random() * jitterMs)));
|
||||
page = await post({ continuation: token });
|
||||
}
|
||||
return cards.slice(0, limit);
|
||||
}
|
||||
|
||||
// Watch-next recommendations use compact renderers or the newer lockup model.
|
||||
// Traverse only the recommendation contents, never player configuration/ads.
|
||||
export function parseRelated(json) {
|
||||
|
||||
@@ -7,7 +7,7 @@ test('parses video renderers into slim cards', () => {
|
||||
const r = parseSearch(fixture);
|
||||
expect(r.length).toBe(4);
|
||||
expect(r[0]).toEqual({
|
||||
id: 'nQWFzMvCfLE', title: 'What A Beautiful Name - Hillsong Worship', channel: 'Hillsong Worship',
|
||||
id: 'nQWFzMvCfLE', url: 'https://www.youtube.com/watch?v=nQWFzMvCfLE', title: 'What A Beautiful Name - Hillsong Worship', channel: 'Hillsong Worship',
|
||||
channelId: 'UC4q12NoPNySbVqwpw4iO5Vg', channelUrl: 'https://www.youtube.com/channel/UC4q12NoPNySbVqwpw4iO5Vg',
|
||||
duration: 343, thumbnail: 'https://i.ytimg.com/vi/nQWFzMvCfLE/mqdefault.jpg',
|
||||
});
|
||||
@@ -60,3 +60,18 @@ test('related HTTP failures and unsupported shapes throw for search fallback', a
|
||||
expect(() => parseRelated({ error: {} })).toThrow('response shape');
|
||||
await expect(related('aaaaaaaaaaa', { fetchImpl: async () => ({ ok: false, status: 429 }) })).rejects.toThrow('429');
|
||||
});
|
||||
|
||||
test('channel and playlist browse parses uploads and resolves handle URLs', async () => {
|
||||
const { browseUploads, parseBrowseVideos } = await import('./innertube.js');
|
||||
const renderer = { videoRenderer: { videoId: 'abcdefghijk', title: { simpleText: 'Upload' }, ownerText: { runs: [{ text: 'Choir', navigationEndpoint: { browseEndpoint: { browseId: 'UC1234567890123456789012', canonicalBaseUrl: '/@choir' } } }] }, lengthText: { simpleText: '2:10' } } };
|
||||
expect(parseBrowseVideos({ contents: [{ gridVideoRenderer: { videoId: 'abcdefghijk', title: { simpleText: 'Upload' } } }] })[0].title).toBe('Upload');
|
||||
const calls = [];
|
||||
const fetchImpl = async (url, init = {}) => {
|
||||
calls.push([String(url), init.body && JSON.parse(init.body)]);
|
||||
if (String(url).includes('/@choir')) return new Response('"browseId":"UC1234567890123456789012"');
|
||||
return Response.json({ contents: [{ gridVideoRenderer: renderer.videoRenderer }] });
|
||||
};
|
||||
const result = await browseUploads('https://www.youtube.com/@choir', { fetchImpl, limit: 5 });
|
||||
expect(result[0].id).toBe('abcdefghijk');
|
||||
expect(calls[1][1].browseId).toBe('UU1234567890123456789012');
|
||||
});
|
||||
|
||||
@@ -158,7 +158,7 @@ function metaFromInfo(info, duration) {
|
||||
// getInfo async (videoId) => yt-dlp -J info — duration, live flags, meta
|
||||
// ----------------------------------------------------------------------------
|
||||
export function createMediaCache({
|
||||
dir, db, download, getInfo,
|
||||
dir, db, download, getInfo, getSourceUrl = id => `https://www.youtube.com/watch?v=${id}`,
|
||||
ffmpeg = 'ffmpeg', ffprobe = 'ffprobe', nice = 'nice',
|
||||
maxBytes = 10 * 1024 ** 3,
|
||||
minFreeBytes = 5 * 1024 ** 3,
|
||||
@@ -314,7 +314,9 @@ export function createMediaCache({
|
||||
await job.persisted;
|
||||
try {
|
||||
await setStatus(job, 'downloading');
|
||||
const info = await getInfo(id);
|
||||
const sourceUrl = await getSourceUrl(id);
|
||||
if (!sourceUrl || !/^https:\/\/www\.youtube\.com\/watch\?v=[\w-]{11}$/.test(sourceUrl)) throw new MediaSkip('source URL is unavailable');
|
||||
const info = await getInfo(id, sourceUrl);
|
||||
const live = info.is_live || ['is_live', 'post_live', 'is_upcoming'].includes(info.live_status);
|
||||
if (live) throw new MediaSkip('live streams are not cached');
|
||||
const expected = typeof info.duration === 'number' ? info.duration : 0;
|
||||
@@ -324,7 +326,7 @@ export function createMediaCache({
|
||||
}
|
||||
await makeRoom(id, Math.max(expected, 60) * EST_BYTES_PER_SEC);
|
||||
|
||||
const raw = await download(id, `${base}.dl.mp4`);
|
||||
const raw = await download(id, `${base}.dl.mp4`, sourceUrl);
|
||||
await setStatus(job, 'validating');
|
||||
|
||||
// Normalise: first video + first audio, moov atom up front so the copy
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Uses real ffmpeg fixtures and the real db.js against a throwaway DB file.
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, rmSync, copyFileSync, writeFileSync, readFileSync, existsSync, readdirSync } from 'node:fs';
|
||||
import { mkdtempSync, rmSync, copyFileSync, writeFileSync, readFileSync, existsSync, readdirSync, unlinkSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
@@ -93,6 +93,7 @@ let dirN = 0;
|
||||
function makeCache({ fixture = 'good.mp4', duration = 10, ...opts } = {}) {
|
||||
const dir = join(root, 'media' + dirN++);
|
||||
const calls = [];
|
||||
const sources = [];
|
||||
const db = {
|
||||
getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia,
|
||||
listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia,
|
||||
@@ -102,13 +103,13 @@ function makeCache({ fixture = 'good.mp4', duration = 10, ...opts } = {}) {
|
||||
const cache = createMediaCache({
|
||||
dir, db, log: silentLog,
|
||||
getInfo: async (id) => ({ id, title: 'T ' + id, channel: 'C', duration }),
|
||||
download: async (id, out) => { calls.push(id); await sleep(30); copyFileSync(fx(state.fixture), out); return out; },
|
||||
download: async (id, out, sourceUrl) => { calls.push(id); sources.push({ id, sourceUrl }); await sleep(30); copyFileSync(fx(state.fixture), out); return out; },
|
||||
transcode: { enabled: false },
|
||||
freeBytes: () => 100 * 1024 ** 3,
|
||||
backfillDelayMs: -1,
|
||||
...opts,
|
||||
});
|
||||
return { cache, dir, calls, state };
|
||||
return { cache, dir, calls, sources, state };
|
||||
}
|
||||
async function clearDb() {
|
||||
for (const r of await dbmod.listMedia()) await dbmod.deleteMedia(r.video_id);
|
||||
@@ -233,6 +234,19 @@ describe('media cache jobs', () => {
|
||||
expect(calls.length).toBe(2);
|
||||
});
|
||||
|
||||
test('missing cached files are re-fetched from the stored source URL', async () => {
|
||||
await clearDb();
|
||||
const id = 'srcAAAAAA01';
|
||||
const sourceUrl = `https://www.youtube.com/watch?v=${id}`;
|
||||
const { cache, dir, calls, sources } = makeCache({ getSourceUrl: async () => sourceUrl });
|
||||
await cache.init();
|
||||
const first = await cache.ensureCached(id, { priority: HIGH });
|
||||
unlinkSync(join(dir, `${id}.${first.gen}.mp4`));
|
||||
await cache.ensureCached(id, { priority: HIGH });
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(sources.every(call => call.id === id && call.sourceUrl === sourceUrl)).toBe(true);
|
||||
});
|
||||
|
||||
test('auto jobs respect the auto length limit; explicit saves do not', async () => {
|
||||
await clearDb();
|
||||
const { cache } = makeCache({ duration: 7200, autoMaxSeconds: 3600 });
|
||||
|
||||
@@ -314,6 +314,7 @@ export function verifyAdminCookie(secret, value, nowSec = Math.floor(Date.now()
|
||||
export function registerNoteRoutes(app, deps) {
|
||||
const {
|
||||
db, getProfile, profileNameRe, runYtdlp, adminPassword, backupDir, adminHtmlPath, workerToken,
|
||||
catalog = null, media = null,
|
||||
} = deps;
|
||||
const ADMIN_COOKIE = 'ytp_admin';
|
||||
const ADMIN_TTL = 30 * 24 * 3600;
|
||||
@@ -695,6 +696,7 @@ export function registerNoteRoutes(app, deps) {
|
||||
const n = await db.getNotes(r.video_id);
|
||||
out.push({
|
||||
id: r.video_id, cacheStatus: r.status, title: meta.title || '', channel: meta.channel || meta.uploader || '',
|
||||
sourceUrl: catalog ? await catalog.getSourceUrl(r.video_id) : null,
|
||||
duration: Number(r.duration) || 0, lastAccess: Number(r.last_access) || 0,
|
||||
lyricsRev: n.lyrics ? n.lyrics.rev : 0, lyricsLines: n.lyrics ? n.lyrics.data.lines.length : 0,
|
||||
});
|
||||
@@ -703,6 +705,12 @@ export function registerNoteRoutes(app, deps) {
|
||||
return c.json({ ok: true, media: out });
|
||||
});
|
||||
|
||||
app.post('/api/admin/media/:id/redownload', requireAdminOrToken, async c => {
|
||||
if (!media) return c.json({ ok: false, error: 'media cache unavailable' }, 503);
|
||||
try { return c.json({ ok: true, ...(await media.redownload(c.req.param('id'))) }, 202); }
|
||||
catch (err) { return c.json({ ok: false, error: err.message }, 400); }
|
||||
});
|
||||
|
||||
app.get('/api/admin/notes/recent', requireAdmin, async (c) => c.json({ ok: true, revs: await db.recentNoteRevs(150) }));
|
||||
|
||||
app.post('/api/admin/notes/:id/:kind/restore', requireAdmin, async (c) => {
|
||||
|
||||
@@ -57,6 +57,8 @@ beforeAll(async () => {
|
||||
backupDir: join(root, 'backups'),
|
||||
adminHtmlPath: htmlPath,
|
||||
workerToken: 'w'.repeat(32),
|
||||
catalog: { getSourceUrl: async id => /^[\w-]{11}$/.test(id) ? `https://www.youtube.com/watch?v=${id}` : null },
|
||||
media: { redownload: async id => ({ status: `queued:${id}` }) },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -278,5 +280,7 @@ test('admin song list includes metadata-only phone songs for lyrics lookup', asy
|
||||
const cookie = await adminCookie();
|
||||
const res = await app.request('/api/admin/media', { headers: { Cookie: cookie } });
|
||||
const song = (await res.json()).media.find(x => x.id === id);
|
||||
expect(song).toMatchObject({ title: 'Phone song', channel: 'Phone artist', duration: 240, cacheStatus: 'failed' });
|
||||
expect(song).toMatchObject({ title: 'Phone song', channel: 'Phone artist', duration: 240, cacheStatus: 'failed', sourceUrl: 'https://www.youtube.com/watch?v=phoneSong01' });
|
||||
expect((await app.request(`/api/admin/media/${id}/redownload`, { method: 'POST', headers: { Cookie: cookie } })).status).toBe(202);
|
||||
expect((await app.request('/api/admin/media/phoneSong01/redownload', { method: 'POST' })).status).toBe(401);
|
||||
});
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"scripts": {
|
||||
"start": "bun server.js",
|
||||
"dev": "bun --hot server.js",
|
||||
"test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js"
|
||||
"test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.14.0",
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// repeat query — from any device, or after a restart — never waits on YouTube.
|
||||
// Capped by row count (default 250,000 queries) AND bytes (default 2 GiB), LRU.
|
||||
import { db } from './db.js';
|
||||
import { ingest, withLocalThumbnails, trimCatalog } from './video-catalog.js';
|
||||
import { ingest, withLocalThumbnails, trimCatalog, foldSearchText } from './video-catalog.js';
|
||||
|
||||
export const MAX_ROWS = Number(process.env.SEARCH_CACHE_MAX_ROWS) || 250_000;
|
||||
export const MAX_BYTES = Number(process.env.SEARCH_CACHE_MAX_BYTES) || 2 * 1024 ** 3;
|
||||
@@ -70,15 +70,28 @@ export async function rememberVideos(cards) {
|
||||
|
||||
// Cards whose title/channel contain EVERY word of the query, most-seen first.
|
||||
export async function searchVideos(q, limit = 60) {
|
||||
const words = String(q).toLowerCase().split(/\s+/).filter(Boolean).slice(0, 8);
|
||||
const folded = foldSearchText(q).trim();
|
||||
const words = folded.split(/\s+/).filter(Boolean).slice(0, 8);
|
||||
if (!words.length) return [];
|
||||
const like = (w) => '%' + w.replace(/[\\%_]/g, '\\$&') + '%';
|
||||
const r = await db.execute({
|
||||
sql: `SELECT card FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')}
|
||||
sql: `SELECT card,seen FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')}
|
||||
ORDER BY seen DESC, updated_at DESC LIMIT ?`,
|
||||
args: [...words.map(like), limit],
|
||||
args: [...words.map(like), Math.max(limit * 10, 200)],
|
||||
});
|
||||
return withLocalThumbnails(r.rows.map((x) => { try { return JSON.parse(x.card); } catch { return null; } }).filter(Boolean));
|
||||
const titleHasAll = title => words.every(word => foldSearchText(title).includes(word));
|
||||
const phrase = folded;
|
||||
const ranked = r.rows.map(row => {
|
||||
try {
|
||||
const card = JSON.parse(row.card);
|
||||
const title = foldSearchText(card.title);
|
||||
const channel = foldSearchText(card.channel);
|
||||
const score = (title === phrase || title.includes(phrase) ? 4 : 0) +
|
||||
(titleHasAll(card.title) ? 2 : 0) + (words.some(word => channel.includes(word)) ? 1 : 0);
|
||||
return { card: { ...card, url: card.url || `https://www.youtube.com/watch?v=${card.id}` }, score, seen: Number(row.seen) || 0 };
|
||||
} catch { return null; }
|
||||
}).filter(Boolean).sort((a, b) => b.score - a.score || b.seen - a.seen).slice(0, limit).map(x => x.card);
|
||||
return withLocalThumbnails(ranked);
|
||||
}
|
||||
|
||||
export async function trimVideos(max = MAX_VIDEOS) {
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import { test, expect } from 'bun:test';
|
||||
import * as sc from './search-cache.js';
|
||||
import { initDb } from './db.js';
|
||||
|
||||
import { test, expect, afterAll } from 'bun:test';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
const root = mkdtempSync(join(tmpdir(), 'ytp-search-cache-'));
|
||||
process.env.DB_PATH = join(root, 'test.db');
|
||||
const sc = await import('./search-cache.js');
|
||||
const { initDb } = await import('./db.js');
|
||||
await initDb();
|
||||
afterAll(() => rmSync(root, { recursive: true, force: true }));
|
||||
const cards = (n) => Array.from({ length: n }, (_, i) => ({ id: 'v' + i, title: 't' + i }));
|
||||
|
||||
test('put/get round-trips, case-insensitive, capped at 200', async () => {
|
||||
@@ -21,11 +26,11 @@ test('trim evicts least-recently-used rows over the row cap', async () => {
|
||||
|
||||
test('known videos are remembered and found by every word', async () => {
|
||||
await sc.rememberVideos([
|
||||
{ id: 'a1', title: 'Way Maker - Sinach', channel: 'Worship' },
|
||||
{ id: 'b2', title: 'Way Maker (live)', channel: 'Other' },
|
||||
{ id: 'c3', title: 'Goodness of God', channel: 'Worship' },
|
||||
{ id: 'aaa11111111', title: 'Way Maker - Sinach', channel: 'Worship' },
|
||||
{ id: 'bbb22222222', title: 'Way Maker (live)', channel: 'Other' },
|
||||
{ id: 'ccc33333333', title: 'Goodness of God', channel: 'Worship' },
|
||||
]);
|
||||
expect((await sc.searchVideos('way maker')).map((c) => c.id).sort()).toEqual(['a1', 'b2']);
|
||||
expect((await sc.searchVideos('maker worship')).map((c) => c.id)).toEqual(['a1']);
|
||||
expect((await sc.searchVideos('way maker')).map((c) => c.id).sort()).toEqual(['aaa11111111', 'bbb22222222']);
|
||||
expect((await sc.searchVideos('maker worship')).map((c) => c.id)).toEqual(['aaa11111111']);
|
||||
expect(await sc.searchVideos('100%')).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -68,6 +68,7 @@ import { registerCatalogRoutes } from './recommendations.js';
|
||||
import QRCode from 'qrcode';
|
||||
import { createYtdlpPool } from './ytdlp-pool.js';
|
||||
import { createWarmQueue } from './warm-queue.js';
|
||||
import { registerDeepSearchRoutes } from './deep-search.js';
|
||||
import { dirname, join as pathJoin } from 'node:path';
|
||||
|
||||
// A media proxy must not die because one client's stream hit an edge case
|
||||
@@ -268,6 +269,7 @@ function slimEntry(j) {
|
||||
return {
|
||||
id,
|
||||
title: pick(j, 'title') || '(untitled)',
|
||||
url: /^[\w-]{11}$/.test(id) ? `https://www.youtube.com/watch?v=${id}` : null,
|
||||
channel: pickChannel(j),
|
||||
channelId: pickChannelId(j),
|
||||
channelUrl: pickChannelUrl(j),
|
||||
@@ -418,6 +420,8 @@ const searchLibrary = async (q) => {
|
||||
// This server's own library first — it still answers when YouTube is unreachable.
|
||||
try { return (await notesDb.listUploads({ q, limit: 20, listedOnly: true })).map(uploads.card); } catch { return []; }
|
||||
};
|
||||
const withYoutubeUrl = card => card && /^[\w-]{11}$/.test(card.id || '')
|
||||
? { ...card, url: `https://www.youtube.com/watch?v=${card.id}` } : card;
|
||||
|
||||
// GET /api/search/local?q=<query> — videos this server already knows (from every
|
||||
// earlier search) plus its own uploads; answers in milliseconds so the app can
|
||||
@@ -426,7 +430,7 @@ app.get('/api/search/local', async (c) => {
|
||||
const q = (c.req.query('q') || '').trim();
|
||||
if (!q) return c.json({ ok: false, error: 'empty query' }, 400);
|
||||
const [mine, known] = await Promise.all([searchLibrary(q), searchCacheDb.searchVideos(q).catch(() => [])]);
|
||||
return c.json({ ok: true, results: [...mine, ...known], local: true });
|
||||
return c.json({ ok: true, results: [...mine, ...known].map(withYoutubeUrl), local: true });
|
||||
});
|
||||
|
||||
// GET /api/search?q=<query>[&refresh=1]
|
||||
@@ -439,7 +443,7 @@ app.get('/api/search', async (c) => {
|
||||
if (!q) return c.json({ ok: false, error: 'empty query' }, 400);
|
||||
const key = q.toLowerCase();
|
||||
const mine = await searchLibrary(q);
|
||||
const answer = (yt, extra = {}) => c.json({ ok: true, results: [...mine, ...yt], ...extra });
|
||||
const answer = (yt, extra = {}) => c.json({ ok: true, results: [...mine, ...yt].map(withYoutubeUrl), ...extra });
|
||||
if (c.req.query('refresh') !== '1') {
|
||||
let hit = searchCache.get(key);
|
||||
if (!hit) {
|
||||
@@ -1101,7 +1105,7 @@ const SAVE_DEFAULT_FORMAT =
|
||||
// again) share ONE yt-dlp run instead of spawning another each time.
|
||||
const inflightDownloads = new Map(); // key -> { promise, waiters, tmpBase }
|
||||
|
||||
async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) {
|
||||
async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal, sourceUrl = null) {
|
||||
await assertNotLive(videoId);
|
||||
const key = videoId + '|' + formatArgs.join(' ');
|
||||
let entry = inflightDownloads.get(key);
|
||||
@@ -1110,10 +1114,11 @@ async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) {
|
||||
const tmp = `${tmpdir()}/${tmpBase}.mp4`;
|
||||
const ctl = new AbortController();
|
||||
entry = { waiters: 0, tmpBase, ctl };
|
||||
entry.promise = withSaveSlot(() => {
|
||||
entry.promise = withSaveSlot(async () => {
|
||||
if (ctl.signal.aborted) throw new Error('save cancelled');
|
||||
const url = sourceUrl || await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`;
|
||||
return runYtdlpResilient([
|
||||
`https://www.youtube.com/watch?v=${videoId}`,
|
||||
url,
|
||||
'--no-warnings', '--no-playlist',
|
||||
...formatArgs,
|
||||
'--limit-rate', DOWNLOAD_RATE,
|
||||
@@ -1183,14 +1188,16 @@ async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal, cachedSrc
|
||||
try {
|
||||
// 1) Grab the full source (video+audio merged) so ffmpeg has both streams
|
||||
// — unless the server cache already holds it.
|
||||
if (!cachedSrc) await withSaveSlot(() => runYtdlpResilient([
|
||||
`https://www.youtube.com/watch?v=${videoId}`,
|
||||
'--no-warnings', '--no-playlist',
|
||||
'-f', SAVE_MUX_FORMAT,
|
||||
'--merge-output-format', 'mp4',
|
||||
'--limit-rate', DOWNLOAD_RATE,
|
||||
'-o', srcTmp,
|
||||
], { signal }));
|
||||
if (!cachedSrc) await withSaveSlot(async () => {
|
||||
const url = await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`;
|
||||
return runYtdlpResilient([
|
||||
url, '--no-warnings', '--no-playlist',
|
||||
'-f', SAVE_MUX_FORMAT,
|
||||
'--merge-output-format', 'mp4',
|
||||
'--limit-rate', DOWNLOAD_RATE,
|
||||
'-o', srcTmp,
|
||||
], { signal });
|
||||
});
|
||||
// 2) Trim + concat the keep segments into the final custom video.
|
||||
await runFfmpeg([
|
||||
'-y', '-hide_banner', '-loglevel', 'error',
|
||||
@@ -1240,6 +1247,7 @@ if (!MEDIA_VOLUME_MARKER || existsSync(MEDIA_VOLUME_MARKER)) mkdirSync(MEDIA_DIR
|
||||
const media = createMediaCache({
|
||||
dir: MEDIA_DIR,
|
||||
volumeMarker: MEDIA_VOLUME_MARKER,
|
||||
getSourceUrl: videoCatalog.getSourceUrl,
|
||||
db: {
|
||||
getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats,
|
||||
// Retention (docs/p2p-architecture.md flow 9): cold copies go before popular ones.
|
||||
@@ -1248,9 +1256,9 @@ const media = createMediaCache({
|
||||
}),
|
||||
},
|
||||
getInfo: async (videoId) => (await resolveStreams(videoId)).info,
|
||||
download: async (videoId, out) => {
|
||||
download: async (videoId, out, sourceUrl) => {
|
||||
await withSaveSlot(() => runYtdlpResilient([
|
||||
`https://www.youtube.com/watch?v=${videoId}`,
|
||||
sourceUrl || `https://www.youtube.com/watch?v=${videoId}`,
|
||||
'--no-warnings', '--no-playlist',
|
||||
'-f', SAVE_MUX_FORMAT,
|
||||
'--merge-output-format', 'mp4',
|
||||
@@ -2291,6 +2299,8 @@ app.get('/api/remote/qr/:code', async (c) => {
|
||||
// ============================================================================
|
||||
const notes = registerNoteRoutes(app, {
|
||||
db: notesDb,
|
||||
catalog: videoCatalog,
|
||||
media,
|
||||
getProfile,
|
||||
profileNameRe: PROFILE_NAME_RE,
|
||||
runYtdlp: runYtdlpResilient,
|
||||
@@ -2304,6 +2314,9 @@ const notes = registerNoteRoutes(app, {
|
||||
{ label: 'Database', path: dirname(process.env.DB_PATH || './data/ytplayer.db') },
|
||||
] },
|
||||
});
|
||||
const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, autoStart: false,
|
||||
searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads,
|
||||
ingest: collectVideoMetadata, putSearch: searchCacheDb.put });
|
||||
|
||||
// ============================================================================
|
||||
// Uploads — the server's own searchable media library (see uploads.js)
|
||||
@@ -2515,6 +2528,7 @@ app.get('/*', indexHtml);
|
||||
// ============================================================================
|
||||
async function main() {
|
||||
await initDb();
|
||||
deepSearch.start();
|
||||
startThumbnails();
|
||||
await p2pDb.initP2pSchema();
|
||||
console.log(`[ytplayer] DB ready`);
|
||||
|
||||
@@ -3,11 +3,18 @@ import { db } from './db.js';
|
||||
import { mergeMissing } from './media-meta-core.js';
|
||||
|
||||
const VIDEO_ID = /^[\w-]{11}$/;
|
||||
const SOURCES = new Set(['search', 'search-cache', 'client-search', 'channel', 'streams', 'playlist', 'profile', 'sync', 'related', 'backfill', 'collector', 'device-play']);
|
||||
const SOURCES = new Set(['search', 'search-cache', 'client-search', 'channel', 'streams', 'playlist', 'profile', 'sync', 'related', 'backfill', 'collector', 'deep', 'device-play']);
|
||||
const text = (v, n = 300) => typeof v === 'string' ? v.trim().slice(0, n) : '';
|
||||
const positive = v => Number.isFinite(Number(v)) && Number(v) > 0 ? Number(v) : 0;
|
||||
const canonicalThumb = id => `https://i.ytimg.com/vi/${id}/hqdefault.jpg`;
|
||||
const canonicalUrl = id => `https://www.youtube.com/watch?v=${id}`;
|
||||
export const foldSearchText = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
|
||||
export const channelKey = c => String(c.channelId || c.channel || '').toLowerCase().replace(/\s*-\s*topic$|vevo$|\s+official$/g, '').trim();
|
||||
export async function getSourceUrl(id) {
|
||||
if (!VIDEO_ID.test(String(id || ''))) return null;
|
||||
const row = (await db.execute({ sql: 'SELECT url FROM video_meta WHERE id=?', args: [id] })).rows[0];
|
||||
return row?.url || `https://www.youtube.com/watch?v=${id}`;
|
||||
}
|
||||
function safeThumbnail(url) {
|
||||
try { const u = new URL(url); return u.protocol === 'https:' && /^(?:i|i\d)\.ytimg\.com$/.test(u.hostname) && !u.port && !u.username && !u.password; }
|
||||
catch { return false; }
|
||||
@@ -23,14 +30,16 @@ export function normalizeCard(v) {
|
||||
// Never fetch arbitrary client URLs (including redirects or private hosts).
|
||||
if (safeThumbnail(u.href)) thumbnail = u.href;
|
||||
} catch { /* canonical YouTube art */ }
|
||||
const card = { id: v.id, title, channel: text(v.channel), channelId: text(v.channelId, 64),
|
||||
const card = { id: v.id, title, url: canonicalUrl(v.id), channel: text(v.channel), channelId: text(v.channelId ?? v.channel_id, 64),
|
||||
duration: positive(v.duration), thumbnail };
|
||||
try {
|
||||
const u = new URL(v.channelUrl);
|
||||
const u = new URL(v.channelUrl || v.channel_url);
|
||||
if (u.protocol === 'https:' && /^(www\.)?youtube\.com$/.test(u.hostname)) card.channelUrl = u.href;
|
||||
} catch { /* optional */ }
|
||||
for (const key of ['tags', 'categories']) if (Array.isArray(v[key])) card[key] = v[key].slice(0, 30).map(x => text(x, 80)).filter(Boolean);
|
||||
if (v.description) card.description = text(v.description, 1200);
|
||||
if (v.published || v.publishedAt || v.upload_date) card.published = text(v.published || v.publishedAt || v.upload_date, 40);
|
||||
if (v.lang || v.language) card.lang = text(v.lang || v.language, 24);
|
||||
if (positive(v.viewCount ?? v.view_count)) card.viewCount = positive(v.viewCount ?? v.view_count);
|
||||
return card;
|
||||
}
|
||||
@@ -67,9 +76,12 @@ async function ingestBatch(cards, source, fillMissing) {
|
||||
await db.batch(batch.flatMap(c => {
|
||||
// Sparse playlist cards must not erase richer extractor metadata.
|
||||
const merged = fillMissing ? mergeMissing(old.get(c.id) || {}, c) : { ...(old.get(c.id) || {}), ...Object.fromEntries(Object.entries(c).filter(([, v]) => v !== '' && v !== 0 && (!Array.isArray(v) || v.length))) };
|
||||
return [{ sql: `INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,1,?)
|
||||
ON CONFLICT(id) DO UPDATE SET card=excluded.card,hay=excluded.hay,seen=seen+1,updated_at=excluded.updated_at`,
|
||||
args: [c.id, JSON.stringify(merged), `${merged.title} ${merged.channel || ''} ${(merged.tags || []).join(' ')}`.toLowerCase(), now] },
|
||||
const hay = foldSearchText([merged.title, merged.channel, merged.channelId, merged.description,
|
||||
...(merged.tags || []), ...(merged.categories || [])].filter(Boolean).join(' '));
|
||||
return [{ sql: `INSERT INTO video_meta (id,card,hay,seen,updated_at,url,channel_id,channel_url,published,lang) VALUES (?,?,?,1,?,?,?,?,?,?)
|
||||
ON CONFLICT(id) DO UPDATE SET card=excluded.card,hay=excluded.hay,seen=seen+1,updated_at=excluded.updated_at,
|
||||
url=excluded.url,channel_id=excluded.channel_id,channel_url=excluded.channel_url,published=excluded.published,lang=excluded.lang`,
|
||||
args: [c.id, JSON.stringify(merged), hay, now, merged.url, merged.channelId || null, merged.channelUrl || null, merged.published || null, merged.lang || null] },
|
||||
{ sql: `INSERT INTO video_meta_sources (video_id,source,last_seen) VALUES (?,?,?)
|
||||
ON CONFLICT(video_id,source) DO UPDATE SET discoveries=discoveries+1,last_seen=excluded.last_seen`, args: [c.id, source, now] },
|
||||
{ sql: `INSERT INTO video_thumbnails (video_id,url) VALUES (?,?) ON CONFLICT(video_id) DO UPDATE SET retry_at=0
|
||||
|
||||
53
server/video-catalog.test.js
Normal file
53
server/video-catalog.test.js
Normal file
@@ -0,0 +1,53 @@
|
||||
import { test, expect } from 'bun:test';
|
||||
import { mkdtempSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const root = mkdtempSync(join(tmpdir(), 'ytp-video-catalog-'));
|
||||
process.env.DB_PATH = join(root, 'test.db');
|
||||
const db = await import('./db.js');
|
||||
const catalog = await import('./video-catalog.js');
|
||||
await db.initDb();
|
||||
|
||||
test('catalog migration backfills canonical URLs and rich metadata, excluding upload ids', async () => {
|
||||
await db.db.execute('DELETE FROM catalog_schema_migrations WHERE version=1');
|
||||
await db.db.execute({ sql: 'INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,?,?)',
|
||||
args: ['abcDEF12345', JSON.stringify({ id: 'abcDEF12345', title: 'Café Worship', channel: 'Choir', channelId: 'UC123', channelUrl: 'https://www.youtube.com/@choir', tags: ['live'], publishedAt: '2024-01-01', language: 'en' }), 'cafe worship choir', 1, 1] });
|
||||
await db.db.execute({ sql: 'INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,?,?)',
|
||||
args: ['upl_abcdef12', JSON.stringify({ id: 'upl_abcdef12', title: 'Upload' }), 'upload', 1, 1] });
|
||||
await db.migrateVideoMeta({ pageSize: 1 });
|
||||
await db.migrateVideoMeta({ pageSize: 1 });
|
||||
const youtube = (await db.db.execute("SELECT * FROM video_meta WHERE id='abcDEF12345'")).rows[0];
|
||||
const upload = (await db.db.execute("SELECT * FROM video_meta WHERE id='upl_abcdef12'")).rows[0];
|
||||
expect(youtube.url).toBe('https://www.youtube.com/watch?v=abcDEF12345');
|
||||
expect(youtube.channel_id).toBe('UC123');
|
||||
expect(youtube.channel_url).toBe('https://www.youtube.com/@choir');
|
||||
expect(youtube.published).toBe('2024-01-01');
|
||||
expect(youtube.lang).toBe('en');
|
||||
expect(youtube.hay).toContain('cafe');
|
||||
expect(JSON.parse(youtube.card).url).toBe(youtube.url);
|
||||
expect(upload.url).toBeNull();
|
||||
});
|
||||
|
||||
test('ingest stores canonical source URL and channel metadata on the card and row', async () => {
|
||||
await catalog.ingest([{ id: 'xyzABC12345', title: 'Música', channel: 'Choir', channel_id: 'UC456', channel_url: 'https://www.youtube.com/@choir', tags: ['Praise'], description: 'Sunday service' }], 'deep');
|
||||
const row = (await db.db.execute("SELECT * FROM video_meta WHERE id='xyzABC12345'")).rows[0];
|
||||
const card = JSON.parse(row.card);
|
||||
expect(row.url).toBe('https://www.youtube.com/watch?v=xyzABC12345');
|
||||
expect(row.channel_id).toBe('UC456');
|
||||
expect(card.url).toBe(row.url);
|
||||
expect(row.hay).toContain('musica');
|
||||
expect(row.hay).toContain('praise');
|
||||
});
|
||||
|
||||
test('accent-insensitive matching ranks title phrase ahead of channel match', async () => {
|
||||
await catalog.ingest([
|
||||
{ id: 'phrase12345', title: 'Café Worship Live', channel: 'Other' },
|
||||
{ id: 'channel1234', title: 'Sunday Set', channel: 'Cafe Worship Live' },
|
||||
{ id: 'words123456', title: 'Worship in the Cafe', channel: 'Other' },
|
||||
], 'deep');
|
||||
const search = await import('./search-cache.js');
|
||||
const results = await search.searchVideos('cafe worship live');
|
||||
expect(results[0].id).toBe('phrase12345');
|
||||
expect(results[0].url).toBe('https://www.youtube.com/watch?v=phrase12345');
|
||||
});
|
||||
38
tests/admin-deep-search.spec.js
Normal file
38
tests/admin-deep-search.spec.js
Normal file
@@ -0,0 +1,38 @@
|
||||
const { test, expect } = require('@playwright/test');
|
||||
|
||||
test('admin deep-search tab estimates, starts, controls and shows persisted progress', async ({ page }) => {
|
||||
page.on('pageerror', error => console.error('pageerror:', error.message));
|
||||
const state = { jobs: [], events: [] };
|
||||
await page.route('**/api/**', async route => {
|
||||
const req = route.request(), url = new URL(req.url());
|
||||
let body = { ok: true };
|
||||
if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true };
|
||||
else if (url.pathname === '/api/admin/media') body = { ok: true, media: [] };
|
||||
else if (url.pathname === '/api/uploads') body = { ok: true, uploads: [] };
|
||||
else if (url.pathname === '/api/admin/deep-search/estimate') body = { ok: true, plan: ['worship', 'worship live'], queries: 2, estimatedMinutes: 1, indexSize: 1240, dailyRemaining: 999 };
|
||||
else if (url.pathname === '/api/admin/deep-search/start') {
|
||||
state.events.push('start'); state.jobs = [{ id: 'deep-1', status: 'running', plan: ['worship', 'worship live'], options: {}, queriesDone: 1, queriesTotal: 2, current: 'worship live', newVideos: 24, totalIndexSize: 1264, etaMs: 12000, errors: [], log: [{ at: Date.now(), message: 'Saved 24 results' }] }];
|
||||
body = { ok: true, id: 'deep-1' };
|
||||
} else if (url.pathname === '/api/admin/deep-search/jobs') body = { ok: true, indexSize: 1264, jobs: state.jobs };
|
||||
else if (/\/api\/admin\/deep-search\/deep-1\/(pause|resume|cancel)$/.test(url.pathname)) {
|
||||
const action = url.pathname.split('/').at(-1); state.events.push(action);
|
||||
state.jobs[0].status = action === 'pause' ? 'paused' : action === 'resume' ? 'running' : 'cancelled';
|
||||
body = { ok: true, status: state.jobs[0].status };
|
||||
}
|
||||
await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) });
|
||||
});
|
||||
await page.goto('/admin.html#deep-search');
|
||||
await expect(page.locator('#deepSearchForm')).toBeVisible();
|
||||
await expect(page.locator('#deepSeeds')).toHaveValue(/worship/);
|
||||
await page.locator('#deepEstimate').click();
|
||||
await expect(page.locator('#deepEstimateText')).toContainText('2 unique queries');
|
||||
await page.locator('#deepStart').click();
|
||||
await expect(page.locator('.deep-job')).toContainText('1/2 queries');
|
||||
await page.locator('[data-deep-action="pause"]').click();
|
||||
await expect(page.locator('.deep-job')).toContainText('Paused');
|
||||
await page.locator('[data-deep-action="resume"]').click();
|
||||
await expect(page.locator('.deep-job')).toContainText('Running');
|
||||
await page.locator('[data-deep-action="cancel"]').click();
|
||||
await expect(page.locator('.deep-job')).toContainText('Cancelled');
|
||||
expect(state.events).toEqual(['start', 'pause', 'resume', 'cancel']);
|
||||
});
|
||||
Reference in New Issue
Block a user