Add searchable index health and repair tools

frontend/admin.html

@@ -567,6 +567,15 @@
       </div>
     </section>
 
+    <section id="p-index" class="panel">
+      <h1 class="page">Search index</h1>
+      <p class="lead">Coverage and discovery sources for the local search catalogue. Backfill advances in bounded, resumable pages.</p>
+      <div class="analytics-grid" id="indexMetrics"></div>
+      <div class="card"><div class="row"><h2 style="margin:0">Discovery sources</h2><span class="grow"></span><button class="btn sm" id="indexRefresh" type="button">Refresh</button></div><div class="scroll"><table class="tbl" id="indexSources"></table></div></div>
+      <div class="card"><h2>Deep search</h2><p class="muted" id="indexLastRun">No deep-search run recorded.</p><div id="indexRunDetail" class="muted"></div></div>
+      <div class="card"><div class="row"><h2 style="margin:0">Catalog repair</h2><span class="grow"></span><button class="btn" id="indexBackfill" type="button">Backfill next page</button><button class="btn dng" id="indexCleanup" type="button">Clean stale entries</button></div><p class="muted">Backfill imports older metadata in one small page. Cleanup removes only old catalogue rows with no source, cache copy, or listening history, then prunes orphaned detail records.</p><p id="indexStatus" class="msg" role="status" aria-live="polite"></p><div class="scroll"><table class="tbl" id="indexBackfillTable"></table></div></div>
+    </section>
+
     <!-- ================= LIBRARY ================= -->
     <section id="p-library" class="panel">
       <h1 class="page">Library</h1>
@@ -760,6 +769,7 @@
     reports: '<path d="M5 21V4M5 4h12l-2.5 4L17 12H5"/>',
     quality: '<path d="M4 5h16M4 12h16M4 19h10"/><path d="m17 18 2 2 4-5"/>',
     cache: '<ellipse cx="12" cy="5" rx="8" ry="3"/><path d="M4 5v14c0 1.7 3.6 3 8 3s8-1.3 8-3V5M4 12c0 1.7 3.6 3 8 3s8-1.3 8-3"/>',
+    index: '<path d="M4 19V5M4 19h16M8 15l3-4 3 2 5-7"/><circle cx="19" cy="6" r="1.5"/>',
     library: '<path d="M9 18V6l10-2v12"/><circle cx="6.5" cy="18" r="2.5"/><circle cx="16.5" cy="16" r="2.5"/>',
     peers: '<circle cx="6" cy="12" r="2.5"/><circle cx="18" cy="6" r="2.5"/><circle cx="18" cy="18" r="2.5"/><path d="M8.3 10.9l7.4-3.8M8.3 13.1l7.4 3.8"/>',
     history: '<path d="M3 12a9 9 0 109-9 9 9 0 00-6.4 2.6L3 8"/><path d="M3 3v5h5M12 7v5l3 2"/>',
@@ -768,7 +778,7 @@
     audit: '<path d="M5 4h14v16H5zM8 8h8M8 12h8M8 16h5"/>',
     access: '<circle cx="8" cy="15" r="4"/><path d="M11 12l9-9M16 7l3 3M14 9l2 2"/>',
   };
-  const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
+  const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
   let tab = 'editor';
   function buildTabs() {
     const html = TABS.map(([k, l]) => `<button class="tab" role="tab" data-tab="${k}" aria-selected="false"><svg viewBox="0 0 24 24">${ICONS[k]}</svg><span>${l}</span>${k === 'reports' ? '<span class="badge hidden" data-badge>0</span>' : ''}</button>`).join('');
@@ -785,6 +795,7 @@
     if (name === 'reports') loadReports();
     if (name === 'quality') loadLyricsQuality();
     if (name === 'cache') loadCacheHealth();
+    if (name === 'index') loadIndexHealth();
     if (name === 'history') loadRevs();
     if (name === 'peers') loadP2p();
     if (name === 'library') loadUploads();
@@ -1044,6 +1055,39 @@
     try { const result = await api(path, { method: 'POST', body }); if (!result.ok) throw Error(result.error || 'Action failed.'); $('cacheStatus').className = 'msg ok'; $('cacheStatus').textContent = 'Cache updated.'; await loadCacheHealth(); }
     catch (error) { $('cacheStatus').className = 'msg err'; $('cacheStatus').textContent = error.message; }
   });
+
+  /* ---------------- search index health ---------------- */
+  async function loadIndexHealth() {
+    $('indexSources').innerHTML = '<tr><td class="muted">Loading source coverage…</td></tr>';
+    try {
+      const j = await api('/api/admin/index/health'); if (!j.ok) throw Error(j.error || 'Index health unavailable.');
+      const pct = value => `${Math.round(Number(value || 0) * 100)}%`;
+      $('indexMetrics').innerHTML = [
+        ['Indexed videos', Number(j.total).toLocaleString(), 'Searchable catalogue rows'],
+        ['Canonical URLs', pct(j.urlCoverage), `${Number(j.urlCount).toLocaleString()} of ${Number(j.total).toLocaleString()}`],
+        ['Detailed metadata', pct(j.detailCoverage), `${Number(j.detailCount).toLocaleString()} enriched records`],
+        ['Saved thumbnails', pct(j.thumbnailCoverage), `${Number(j.thumbnailCount).toLocaleString()} stored images`],
+      ].map(([label, value, detail]) => `<div class="metric"><span>${esc(label)}</span><b>${esc(value)}</b><small>${esc(detail)}</small></div>`).join('');
+      $('indexSources').innerHTML = j.sources.length ? `<tr><th>Source</th><th>Videos</th><th>Discoveries</th><th>Last seen</th></tr>${j.sources.map(x => `<tr><td>${esc(x.source)}</td><td>${Number(x.videos).toLocaleString()}</td><td>${Number(x.discoveries).toLocaleString()}</td><td>${when(Math.floor(x.lastSeen / 1000))}</td></tr>`).join('')}` : '<tr><td class="muted">No source records yet.</td></tr>';
+      if (j.lastRun) {
+        $('indexLastRun').textContent = `${j.lastRun.status} · ${new Date(j.lastRun.createdAt).toLocaleString()}${j.lastRun.error ? ` · ${j.lastRun.error}` : ''}`;
+        $('indexRunDetail').textContent = `${Number(j.lastRun.progress.queriesDone || 0)} queries complete · ${Number(j.lastRun.progress.newVideos || 0)} new videos`;
+      } else { $('indexLastRun').textContent = 'No deep-search run recorded.'; $('indexRunDetail').textContent = ''; }
+      $('indexBackfillTable').innerHTML = j.backfill.length ? `<tr><th>Source table</th><th>Progress</th><th>Status</th></tr>${j.backfill.map(x => `<tr><td>${esc(x.source)}</td><td>${Number(x.cursor).toLocaleString()} / ${Number(x.ceiling).toLocaleString()}</td><td>${x.done ? 'Complete' : 'Ready for next page'}</td></tr>`).join('')}` : '<tr><td class="muted">Backfill has not started. Use “Backfill next page” to initialize it.</td></tr>';
+    } catch (error) { $('indexStatus').className = 'msg err'; $('indexStatus').textContent = error.message; }
+  }
+  $('indexRefresh').addEventListener('click', loadIndexHealth);
+  $('indexBackfill').addEventListener('click', async () => {
+    $('indexStatus').className = 'msg'; $('indexStatus').textContent = 'Backfilling one bounded page…';
+    try { const j = await api('/api/admin/index/backfill', { method: 'POST', body: {} }); if (!j.ok) throw Error(j.error || 'Backfill failed.'); $('indexStatus').className = 'msg ok'; $('indexStatus').textContent = `Processed ${j.processed} records. Run again to continue.`; await loadIndexHealth(); }
+    catch (error) { $('indexStatus').className = 'msg err'; $('indexStatus').textContent = error.message; }
+  });
+  $('indexCleanup').addEventListener('click', async () => {
+    if (!confirm('Remove only stale, unreferenced catalogue entries and orphaned details? This cannot be undone.')) return;
+    $('indexStatus').className = 'msg'; $('indexStatus').textContent = 'Cleaning stale entries…';
+    try { const j = await api('/api/admin/index/cleanup', { method: 'POST', body: { days: 365 } }); if (!j.ok) throw Error(j.error || 'Cleanup failed.'); $('indexStatus').className = 'msg ok'; $('indexStatus').textContent = `Removed ${j.removed} stale catalogue entries.`; await loadIndexHealth(); }
+    catch (error) { $('indexStatus').className = 'msg err'; $('indexStatus').textContent = error.message; }
+  });
   $('tokenForm').addEventListener('submit', async (e) => {
     e.preventDefault();
     const label = $('tokenLabel').value.trim();