Add resumable admin deep search
This commit is contained in:
Jonathan Sykes
@@ -280,6 +280,14 @@
|
||||
.collection-job p { margin: 4px 0; color: var(--text-2); }
|
||||
#metadataDetail { white-space: pre-wrap; overflow-wrap: anywhere; max-height: 400px; overflow: auto; font-size: 13px; }
|
||||
@media (max-width: 480px) { .analytics-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } .metric { padding: 12px; } .metric b { font-size: 22px; } }
|
||||
.deep-grid { display: grid; grid-template-columns: repeat(auto-fit,minmax(min(100%,240px),1fr)); gap: 12px; }
|
||||
.deep-grid label { display: grid; gap: 5px; min-width: 0; color: var(--text); font-size: 13px; }
|
||||
.deep-search .muted { color: var(--text-2); }
|
||||
.deep-grid .field { width: 100%; flex: initial; }
|
||||
.deep-grid textarea.field { height: auto; min-height: 110px; padding: 10px 12px; resize: vertical; }
|
||||
.deep-job { padding: 14px 0; border-bottom: 1px solid var(--line-2); overflow-wrap: anywhere; }
|
||||
.deep-job progress { width: 100%; height: 8px; accent-color: var(--a1); }
|
||||
.deep-log { max-height: 240px; overflow: auto; white-space: pre-wrap; font: 12px/1.5 var(--mono); color: var(--text-2); }
|
||||
@media (min-width: 761px) and (max-width: 1100px) {
|
||||
.top { flex-wrap: wrap; }
|
||||
.tabs { order: 2; width: 100%; margin-left: 0; overflow-x: auto; }
|
||||
@@ -547,6 +555,29 @@
|
||||
<div class="card"><h2>Most played</h2><div id="analyticsTop"></div></div>
|
||||
</section>
|
||||
|
||||
<section id="p-deep-search" class="panel deep-search">
|
||||
<h1 class="page">Deep search</h1>
|
||||
<p class="lead">Build the local video index from YouTube searches. Runs are owned by the server and continue if this page closes.</p>
|
||||
<div class="card"><h2>Query plan</h2><p class="muted">One seed per line. Presets include worship terms and artists in English, Tagalog, Cebuano and Spanish.</p>
|
||||
<form id="deepSearchForm">
|
||||
<div class="deep-grid">
|
||||
<label>Seed terms<textarea class="field" id="deepSeeds" required></textarea></label>
|
||||
<label>Modifiers, one per line<textarea class="field" id="deepModifiers"></textarea></label>
|
||||
<label>Maximum queries<input class="field" id="deepMaxQueries" type="number" min="1" max="500" value="160" /></label>
|
||||
<label>Results per query<input class="field" id="deepResults" type="number" min="1" max="200" value="50" /></label>
|
||||
<label>Related videos<select class="field" id="deepDepth"><option value="0">Off</option><option value="1">Top hits · depth 1</option><option value="2">Top hits · depth 2</option></select></label>
|
||||
<label>Related results per query<input class="field" id="deepRelatedTop" type="number" min="1" max="10" value="3" /></label>
|
||||
<label>Channel IDs or URLs<textarea class="field" id="deepChannels" placeholder="UC… or https://www.youtube.com/channel/UC…"></textarea></label>
|
||||
<label>Playlist IDs or URLs<textarea class="field" id="deepPlaylists" placeholder="PL… or https://www.youtube.com/playlist?list=PL…"></textarea></label>
|
||||
<label>Politeness delay (seconds)<input class="field" id="deepDelay" type="number" min="1" max="300" value="10" /></label>
|
||||
<label>Daily query cap<input class="field" id="deepDailyCap" type="number" min="1" max="10000" value="1000" /></label>
|
||||
</div>
|
||||
<div class="row" style="margin-top:12px"><button class="btn" type="button" id="deepEstimate">Estimate plan</button><button class="btn pri" type="submit" id="deepStart">Start deep search</button><span class="muted" id="deepEstimateText" role="status" aria-live="polite"></span></div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="card"><div class="row"><h2 style="margin:0">Runs</h2><span class="grow"></span><button class="btn sm" type="button" id="deepRefresh">Refresh</button></div><p class="muted" id="deepIndexStatus"></p><div id="deepJobs"></div></div>
|
||||
</section>
|
||||
|
||||
<section id="p-access" class="panel">
|
||||
<h1 class="page">API access</h1>
|
||||
<p class="lead">Tokens let scripts write shared lyrics and chapters. A token is shown once — store it in your secrets vault.</p>
|
||||
@@ -608,9 +639,10 @@
|
||||
peers: '<circle cx="6" cy="12" r="2.5"/><circle cx="18" cy="6" r="2.5"/><circle cx="18" cy="18" r="2.5"/><path d="M8.3 10.9l7.4-3.8M8.3 13.1l7.4 3.8"/>',
|
||||
history: '<path d="M3 12a9 9 0 109-9 9 9 0 00-6.4 2.6L3 8"/><path d="M3 3v5h5M12 7v5l3 2"/>',
|
||||
analytics: '<path d="M4 20h16M7 16V9M12 16V4M17 16v-5"/>',
|
||||
'deep-search': '<circle cx="10.5" cy="10.5" r="6.5"/><path d="m15.5 15.5 5 5M8 10.5h5M10.5 8v5"/>',
|
||||
access: '<circle cx="8" cy="15" r="4"/><path d="M11 12l9-9M16 7l3 3M14 9l2 2"/>',
|
||||
};
|
||||
const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['access', 'API']];
|
||||
const TABS = [['editor', 'Editor'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['access', 'API']];
|
||||
let tab = 'editor';
|
||||
function buildTabs() {
|
||||
const html = TABS.map(([k, l]) => `<button class="tab" role="tab" data-tab="${k}" aria-selected="false"><svg viewBox="0 0 24 24">${ICONS[k]}</svg><span>${l}</span>${k === 'reports' ? '<span class="badge hidden" data-badge>0</span>' : ''}</button>`).join('');
|
||||
@@ -630,6 +662,7 @@
|
||||
if (name === 'library') loadUploads();
|
||||
if (name === 'access') loadTokens();
|
||||
if (name === 'analytics') loadAnalytics();
|
||||
if (name === 'deep-search') loadDeepSearch();
|
||||
window.scrollTo({ top: 0 });
|
||||
}
|
||||
function setBadge(n) {
|
||||
@@ -723,6 +756,52 @@
|
||||
$('metadataFilterForm').addEventListener('submit', e => { e.preventDefault(); metadataOffset = 0; loadMetadata(); });
|
||||
$('metadataPrev').addEventListener('click', () => { metadataOffset = Math.max(0, metadataOffset - 50); loadMetadata(); });
|
||||
$('metadataNext').addEventListener('click', () => { metadataOffset += 50; loadMetadata(); });
|
||||
|
||||
/* ---------------- durable deep search ---------------- */
|
||||
let deepTimer = null;
|
||||
const lines = id => $(id).value.split(/\r?\n/).map(s => s.trim()).filter(Boolean);
|
||||
function deepBody() {
|
||||
return { seeds: lines('deepSeeds'), modifiers: lines('deepModifiers'), maxQueries: Number($('deepMaxQueries').value),
|
||||
resultsPerQuery: Number($('deepResults').value), relatedDepth: Number($('deepDepth').value), relatedTop: Number($('deepRelatedTop').value),
|
||||
channels: lines('deepChannels'), playlists: lines('deepPlaylists'), delaySeconds: Number($('deepDelay').value), dailyCap: Number($('deepDailyCap').value) };
|
||||
}
|
||||
if (!$('deepSeeds').value) $('deepSeeds').value = 'worship\npraise\nhymns\ngospel\npagsamba\npapuri\nmga awit ng papuri\nebanghelyo\ndayeg\npagdayeg\nmga alawiton\nmaayong balita\nadoración\nalabanza\nhimnos\nevangelio\nHillsong Worship\nElevation Worship\nBethel Music\nPlanetshakers\nDon Moen\nSandi Patty';
|
||||
if (!$('deepModifiers').value) $('deepModifiers').value = 'lyrics\nlive\nacoustic\ninstrumental\nkaraoke';
|
||||
$('deepEstimate').addEventListener('click', async () => {
|
||||
const label = $('deepEstimateText'); label.textContent = 'Counting the plan…';
|
||||
try { const j = await api('/api/admin/deep-search/estimate', { method: 'POST', body: deepBody() });
|
||||
label.textContent = j.ok ? `${j.queries} unique queries · ${j.browseSources} browse sources · about ${j.estimatedMinutes} min · ${j.indexSize.toLocaleString()} indexed · ${j.dailyRemaining} daily requests remain` : j.error || 'Estimate unavailable';
|
||||
} catch { label.textContent = 'Could not reach the server.'; }
|
||||
});
|
||||
$('deepSearchForm').addEventListener('submit', async e => {
|
||||
e.preventDefault(); const button = $('deepStart'); button.disabled = true;
|
||||
try { const j = await api('/api/admin/deep-search/start', { method: 'POST', body: deepBody() });
|
||||
$('deepEstimateText').textContent = j.ok ? 'Job queued. It will continue if this page closes.' : j.error || 'Could not start job.';
|
||||
await loadDeepSearch();
|
||||
} catch { $('deepEstimateText').textContent = 'Could not reach the server.'; }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
async function loadDeepSearch() {
|
||||
try {
|
||||
const j = await api('/api/admin/deep-search/jobs'); if (!j.ok) throw Error(j.error || 'Could not load deep-search jobs.');
|
||||
$('deepIndexStatus').textContent = `${Number(j.indexSize).toLocaleString()} videos in the local search index.`;
|
||||
$('deepJobs').innerHTML = j.jobs.length ? j.jobs.map(job => {
|
||||
const active = ['queued','running','paused'].includes(job.status), progress = job.queriesTotal ? Math.round(job.queriesDone / job.queriesTotal * 100) : 0;
|
||||
const eta = job.etaMs == null ? 'estimating' : job.etaMs < 60000 ? `${Math.ceil(job.etaMs / 1000)} sec` : `${Math.ceil(job.etaMs / 60000)} min`;
|
||||
const controls = job.status === 'running' || job.status === 'queued' ? `<button class="btn sm" data-deep-action="pause" data-id="${esc(job.id)}">Pause</button>` : job.status === 'paused' ? `<button class="btn sm" data-deep-action="resume" data-id="${esc(job.id)}">Resume</button>` : '';
|
||||
return `<article class="deep-job"><div class="row"><b>${esc(job.status[0].toUpperCase() + job.status.slice(1))}</b><span class="muted">${job.queriesDone}/${job.queriesTotal} queries · ${job.newVideos} new videos · ETA ${eta}</span><span class="grow"></span>${controls}${active ? `<button class="btn sm dng" data-deep-action="cancel" data-id="${esc(job.id)}">Cancel</button>` : ''}</div><progress max="100" value="${progress}" aria-label="Deep search progress"></progress>${job.current ? `<p class="muted">Current: ${esc(job.current)}</p>` : ''}${job.errors?.length ? `<details><summary>${job.errors.length} errors</summary>${job.errors.map(x => `<p>${esc(x.query)}: ${esc(x.error)}</p>`).join('')}</details>` : ''}<details><summary>Live log (${job.log?.length || 0})</summary><div class="deep-log">${(job.log || []).map(x => `${new Date(x.at).toLocaleTimeString()} ${esc(x.message)}`).join('\n')}</div></details></article>`;
|
||||
}).join('') : '<p class="muted">No deep-search runs yet.</p>';
|
||||
clearTimeout(deepTimer); deepTimer = null;
|
||||
if (tab === 'deep-search' && j.jobs.some(job => ['queued','running'].includes(job.status))) deepTimer = setTimeout(loadDeepSearch, 3000);
|
||||
} catch (error) { $('deepIndexStatus').textContent = error.message || 'Could not load deep-search progress.'; }
|
||||
}
|
||||
$('deepRefresh').addEventListener('click', loadDeepSearch);
|
||||
$('deepJobs').addEventListener('click', async e => {
|
||||
const button = e.target.closest('[data-deep-action]'); if (!button) return;
|
||||
button.disabled = true;
|
||||
try { const j = await api(`/api/admin/deep-search/${encodeURIComponent(button.dataset.id)}/${button.dataset.deepAction}`, { method: 'POST', body: {} }); if (!j.ok) $('deepIndexStatus').textContent = j.error || `Could not ${button.dataset.deepAction} job.`; }
|
||||
finally { await loadDeepSearch(); }
|
||||
});
|
||||
$('metadataTable').addEventListener('click', async e => {
|
||||
const btn = e.target.closest('[data-metadata-id]'); if (!btn) return;
|
||||
const j = await api('/api/admin/metadata/' + btn.dataset.metadataId); if (!j.ok) return;
|
||||
@@ -902,7 +981,7 @@
|
||||
const [m, u] = await Promise.all([api('/api/admin/media'), api('/api/uploads?limit=200')]);
|
||||
const seen = new Set();
|
||||
SONGS = [];
|
||||
for (const r of (m.media || [])) { seen.add(r.id); SONGS.push({ id: r.id, title: r.title || r.id, sub: r.channel, lines: r.lyricsLines || 0 }); }
|
||||
for (const r of (m.media || [])) { seen.add(r.id); SONGS.push({ id: r.id, title: r.title || r.id, sub: r.channel, lines: r.lyricsLines || 0, sourceUrl: r.sourceUrl }); }
|
||||
for (const r of (u.uploads || [])) if (!seen.has(r.id)) SONGS.push({ id: r.id, title: r.title, sub: r.artist || 'Upload', lines: -1 });
|
||||
paintSongs();
|
||||
}
|
||||
@@ -912,6 +991,7 @@
|
||||
$('songList').innerHTML = rows.length ? rows.map((r) => `<div class="song-row"><button type="button" class="song-open" data-open="${esc(r.id)}">
|
||||
<span style="min-width:0;flex:1"><b>${esc(r.title)}</b><small>${esc(r.sub || '')}</small></span>
|
||||
<span class="st ${r.lines > 0 ? 'has' : ''}">${r.lines > 0 ? `${r.lines} lines` : r.lines === 0 ? 'no lyrics' : 'open'}</span></button>
|
||||
${r.sourceUrl ? `<a class="btn sm" href="${esc(r.sourceUrl)}" target="_blank" rel="noopener">Source</a><button type="button" class="btn sm" data-redownload="${esc(r.id)}">Re-download from source</button>` : ''}
|
||||
<button type="button" class="btn sm whisper-request" data-whisper="${esc(r.id)}" aria-label="Transcribe ${esc(r.title)} with Whisper">Whisper</button></div>`).join('')
|
||||
: '<div class="muted" style="padding:10px">Nothing matches — paste a link above to open any YouTube song.</div>';
|
||||
paintWhisper();
|
||||
@@ -919,8 +999,17 @@
|
||||
$('songFilter').addEventListener('input', paintSongs);
|
||||
$('songList').addEventListener('click', (e) => {
|
||||
const whisper = e.target.closest('[data-whisper]'); if (whisper) { startWhisper(whisper.dataset.whisper); return; }
|
||||
const source = e.target.closest('[data-redownload]'); if (source) { startSourceRedownload(source); return; }
|
||||
const b = e.target.closest('[data-open]'); if (b) leOpen(b.dataset.open);
|
||||
});
|
||||
async function startSourceRedownload(button) {
|
||||
button.disabled = true; const original = button.textContent; button.textContent = 'Queueing…';
|
||||
try { const result = await api(`/api/admin/media/${encodeURIComponent(button.dataset.redownload)}/redownload`, { method: 'POST', body: {} });
|
||||
button.textContent = result.ok ? 'Queued' : 'Could not queue';
|
||||
button.title = result.error || 'The server-owned download continues if this page closes.';
|
||||
} catch { button.textContent = 'Connection failed'; }
|
||||
setTimeout(() => { button.textContent = original; button.disabled = false; }, 2000);
|
||||
}
|
||||
$('recentChips').addEventListener('click', (e) => { const b = e.target.closest('[data-open]'); if (b) leOpen(b.dataset.open); });
|
||||
|
||||
/* ---------------- p2p ---------------- */
|
||||
|
||||
Reference in New Issue
Block a user