diff --git a/frontend/admin.html b/frontend/admin.html index 291115c..ceb629c 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -567,6 +567,15 @@ +
+

Search index

+

Coverage and discovery sources for the local search catalogue. Backfill advances in bounded, resumable pages.

+
+

Discovery sources

+

Deep search

No deep-search run recorded.

+

Catalog repair

Backfill imports older metadata in one small page. Cleanup removes only old catalogue rows with no source, cache copy, or listening history, then prunes orphaned detail records.

+
+

Library

@@ -760,6 +769,7 @@ reports: '', quality: '', cache: '', + index: '', library: '', peers: '', history: '', @@ -768,7 +778,7 @@ audit: '', access: '', }; - const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -785,6 +795,7 @@ if (name === 'reports') loadReports(); if (name === 'quality') loadLyricsQuality(); if (name === 'cache') loadCacheHealth(); + if (name === 'index') loadIndexHealth(); if (name === 'history') loadRevs(); if (name === 'peers') loadP2p(); if (name === 'library') loadUploads(); @@ -1044,6 +1055,39 @@ try { const result = await api(path, { method: 'POST', body }); if (!result.ok) throw Error(result.error || 'Action failed.'); $('cacheStatus').className = 'msg ok'; $('cacheStatus').textContent = 'Cache updated.'; await loadCacheHealth(); } catch (error) { $('cacheStatus').className = 'msg err'; $('cacheStatus').textContent = error.message; } }); + + /* ---------------- search index health ---------------- */ + async function loadIndexHealth() { + $('indexSources').innerHTML = 'Loading source coverage…'; + try { + const j = await api('/api/admin/index/health'); if (!j.ok) throw Error(j.error || 'Index health unavailable.'); + const pct = value => `${Math.round(Number(value || 0) * 100)}%`; + $('indexMetrics').innerHTML = [ + ['Indexed videos', Number(j.total).toLocaleString(), 'Searchable catalogue rows'], + ['Canonical URLs', pct(j.urlCoverage), `${Number(j.urlCount).toLocaleString()} of ${Number(j.total).toLocaleString()}`], + ['Detailed metadata', pct(j.detailCoverage), `${Number(j.detailCount).toLocaleString()} enriched records`], + ['Saved thumbnails', pct(j.thumbnailCoverage), `${Number(j.thumbnailCount).toLocaleString()} stored images`], + ].map(([label, value, detail]) => `
${esc(label)}${esc(value)}${esc(detail)}
`).join(''); + $('indexSources').innerHTML = j.sources.length ? `SourceVideosDiscoveriesLast seen${j.sources.map(x => `${esc(x.source)}${Number(x.videos).toLocaleString()}${Number(x.discoveries).toLocaleString()}${when(Math.floor(x.lastSeen / 1000))}`).join('')}` : 'No source records yet.'; + if (j.lastRun) { + $('indexLastRun').textContent = `${j.lastRun.status} · ${new Date(j.lastRun.createdAt).toLocaleString()}${j.lastRun.error ? ` · ${j.lastRun.error}` : ''}`; + $('indexRunDetail').textContent = `${Number(j.lastRun.progress.queriesDone || 0)} queries complete · ${Number(j.lastRun.progress.newVideos || 0)} new videos`; + } else { $('indexLastRun').textContent = 'No deep-search run recorded.'; $('indexRunDetail').textContent = ''; } + $('indexBackfillTable').innerHTML = j.backfill.length ? `Source tableProgressStatus${j.backfill.map(x => `${esc(x.source)}${Number(x.cursor).toLocaleString()} / ${Number(x.ceiling).toLocaleString()}${x.done ? 'Complete' : 'Ready for next page'}`).join('')}` : 'Backfill has not started. Use “Backfill next page” to initialize it.'; + } catch (error) { $('indexStatus').className = 'msg err'; $('indexStatus').textContent = error.message; } + } + $('indexRefresh').addEventListener('click', loadIndexHealth); + $('indexBackfill').addEventListener('click', async () => { + $('indexStatus').className = 'msg'; $('indexStatus').textContent = 'Backfilling one bounded page…'; + try { const j = await api('/api/admin/index/backfill', { method: 'POST', body: {} }); if (!j.ok) throw Error(j.error || 'Backfill failed.'); $('indexStatus').className = 'msg ok'; $('indexStatus').textContent = `Processed ${j.processed} records. Run again to continue.`; await loadIndexHealth(); } + catch (error) { $('indexStatus').className = 'msg err'; $('indexStatus').textContent = error.message; } + }); + $('indexCleanup').addEventListener('click', async () => { + if (!confirm('Remove only stale, unreferenced catalogue entries and orphaned details? This cannot be undone.')) return; + $('indexStatus').className = 'msg'; $('indexStatus').textContent = 'Cleaning stale entries…'; + try { const j = await api('/api/admin/index/cleanup', { method: 'POST', body: { days: 365 } }); if (!j.ok) throw Error(j.error || 'Cleanup failed.'); $('indexStatus').className = 'msg ok'; $('indexStatus').textContent = `Removed ${j.removed} stale catalogue entries.`; await loadIndexHealth(); } + catch (error) { $('indexStatus').className = 'msg err'; $('indexStatus').textContent = error.message; } + }); $('tokenForm').addEventListener('submit', async (e) => { e.preventDefault(); const label = $('tokenLabel').value.trim(); diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js index 0ea8003..9ae45ba 100644 --- a/playwright.admin-audit.config.js +++ b/playwright.admin-audit.config.js @@ -1,3 +1,3 @@ const { defineConfig } = require('@playwright/test'); const classic = require('./playwright.classic.config'); -module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache)\.spec\.js/ }); +module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index)\.spec\.js/ }); diff --git a/server/admin-tools.js b/server/admin-tools.js index f44fb9e..fae0f23 100644 --- a/server/admin-tools.js +++ b/server/admin-tools.js @@ -4,7 +4,7 @@ const clientOf = db => typeof db?.execute === 'function' ? db : db?.db; const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } }; const number = value => Number(value) || 0; -export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null }) { +export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null, backfillStep = null }) { const sql = clientOf(db); app.get('/api/admin/lyrics/quality', adminAuth, async c => { @@ -97,4 +97,53 @@ export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, bac return c.json({ ok: true, pinned: body.pinned }); }); + app.get('/api/admin/index/health', adminAuth, async c => { + const [stats, sources, latest, backfill] = await Promise.all([ + sql.execute(`SELECT COUNT(*) AS total,SUM(CASE WHEN url IS NOT NULL AND url<>'' THEN 1 ELSE 0 END) AS urls, + SUM(CASE WHEN EXISTS(SELECT 1 FROM video_details d WHERE d.video_id=video_meta.id) THEN 1 ELSE 0 END) AS details, + SUM(CASE WHEN EXISTS(SELECT 1 FROM video_thumbnails t WHERE t.video_id=video_meta.id AND t.data IS NOT NULL) THEN 1 ELSE 0 END) AS thumbnails FROM video_meta`), + sql.execute('SELECT source,COUNT(*) AS videos,SUM(discoveries) AS discoveries,MAX(last_seen) AS lastSeen FROM video_meta_sources GROUP BY source ORDER BY videos DESC'), + sql.execute('SELECT id,status,created_at,updated_at,state,error FROM deep_search_jobs ORDER BY created_at DESC LIMIT 1'), + sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source'), + ]); + const row = stats.rows[0] || {}, total = number(row.total); + const last = latest.rows[0]; + return c.json({ ok: true, total, urlCount: number(row.urls), detailCount: number(row.details), thumbnailCount: number(row.thumbnails), + urlCoverage: total ? number(row.urls) / total : 0, detailCoverage: total ? number(row.details) / total : 0, thumbnailCoverage: total ? number(row.thumbnails) / total : 0, + sources: sources.rows.map(x => ({ source: x.source, videos: number(x.videos), discoveries: number(x.discoveries), lastSeen: number(x.lastSeen) })), + lastRun: last ? { id: last.id, status: last.status, createdAt: number(last.created_at), updatedAt: number(last.updated_at), error: last.error || null, progress: parse(last.state) } : null, + backfill: backfill.rows.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) }, 200, { 'Cache-Control': 'no-store' }); + }); + + app.post('/api/admin/index/backfill', adminAuth, async c => { + if (!backfillStep) return c.json({ ok: false, error: 'Catalog backfill is unavailable.' }, 503); + const processed = await backfillStep(100); + const state = (await sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source')).rows; + await audit(db, 'admin', 'index.backfill', '', { processed }); + return c.json({ ok: true, processed, backfill: state.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) }); + }); + + app.post('/api/admin/index/cleanup', adminAuth, async c => { + let body = {}; try { body = await c.req.json(); } catch {} + const days = Math.max(180, Math.min(3650, Number(body.days) || 365)); + const cutoff = Date.now() - days * 86400_000; + const candidates = await sql.execute({ sql: `SELECT COUNT(*) AS n FROM video_meta m WHERE m.updated_at c.req.header('x-admin') === 'yes' ? next() : c.j beforeAll(async () => { await db.initDb(); - registerAdminToolsRoutes(app, { db, adminAuth: auth }); + registerAdminToolsRoutes(app, { db, adminAuth: auth, backfillStep: async () => 3 }); await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] }); await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] }); await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] }); await db.db.execute({ sql: "INSERT INTO media_cache(video_id,status,size,gen) VALUES('CacheItem001','ready',700,1),('CacheItem002','ready',400,1)" }); + await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at,url) VALUES(?,?,?,?,?,?)', args: ['IndexSong001', JSON.stringify({ title: 'Indexed song' }), 'indexed song', 1, Date.now(), 'https://www.youtube.com/watch?v=IndexSong001'] }); + await db.db.execute({ sql: "INSERT INTO video_meta_sources(video_id,source,last_seen) VALUES('IndexSong001','deep',?)", args: [Date.now()] }); + await db.db.execute({ sql: "INSERT INTO video_details(video_id,metadata,updated_at) VALUES('IndexSong001','{}',?)", args: [Date.now()] }); + await db.db.execute({ sql: "INSERT INTO video_thumbnails(video_id,url,data,mime,size,fetched_at) VALUES('IndexSong001','https://i.ytimg.com/vi/IndexSong001/hqdefault.jpg',x'01','image/jpeg',1,?)", args: [Date.now()] }); + await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,1)', args: ['OldOrphan01', JSON.stringify({ title: 'Old orphan' }), 'old orphan', 1] }); + await db.db.execute({ sql: "INSERT INTO deep_search_jobs(id,status,plan,options,state,created_at,updated_at) VALUES('job-1','complete','[]','{}','{\"queriesDone\":2,\"newVideos\":5}',1,2)" }); }); afterAll(() => rmSync(root, { recursive: true, force: true })); @@ -27,12 +33,12 @@ describe('lyrics quality queue', () => { expect((await app.request('/api/admin/lyrics/quality')).status).toBe(401); const headers = { 'x-admin': 'yes' }; const missing = await (await app.request('/api/admin/lyrics/quality?category=missing', { headers })).json(); - expect(missing.total).toBe(1); - expect(missing.songs[0]).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] }); + expect(missing.total).toBeGreaterThanOrEqual(1); + expect(missing.songs.find(song => song.id === 'QualMiss001')).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] }); const untimed = await (await app.request('/api/admin/lyrics/quality?category=untimed', { headers })).json(); expect(untimed.songs[0]).toMatchObject({ id: 'Untimely001', lines: 2, untimed: true }); const short = await (await app.request('/api/admin/lyrics/quality?category=short&minLines=3', { headers })).json(); - expect(short.songs[0].reasons).toContain('short'); + expect(short.songs.find(song => song.id === 'Untimely001').reasons).toContain('short'); const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json(); expect(page.songs).toHaveLength(1); }); @@ -48,4 +54,20 @@ describe('lyrics quality queue', () => { const health = await (await app.request('/api/admin/cache/health', { headers: { 'x-admin': 'yes' } })).json(); expect(health.pinnedBytes).toBe(700); }); + + test('index health reports coverage, source and deep-search data; repair actions are audited', async () => { + const headers = { 'x-admin': 'yes', 'content-type': 'application/json' }; + const health = await (await app.request('/api/admin/index/health', { headers })).json(); + expect(health.total).toBeGreaterThanOrEqual(1); + expect(health.urlCount).toBeGreaterThanOrEqual(1); + expect(health.detailCount).toBe(1); + expect(health.thumbnailCount).toBe(1); + expect(health.sources[0].source).toBe('deep'); + expect(health.lastRun.progress.queriesDone).toBe(2); + const step = await (await app.request('/api/admin/index/backfill', { method: 'POST', headers, body: '{}' })).json(); + expect(step.processed).toBe(3); + const clean = await (await app.request('/api/admin/index/cleanup', { method: 'POST', headers, body: JSON.stringify({ days: 180 }) })).json(); + expect(clean.removed).toBeGreaterThanOrEqual(1); + expect((await db.db.execute({ sql: "SELECT COUNT(*) AS n FROM video_meta WHERE id='OldOrphan01'" })).rows[0].n).toBe(0); + }); }); diff --git a/server/server.js b/server/server.js index 096869c..7e6d3ac 100644 --- a/server/server.js +++ b/server/server.js @@ -64,7 +64,7 @@ import { sha256Range } from './hash.js'; import * as innertube from './innertube.js'; import { createRelatedService, registerRelatedRoute } from './related.js'; import * as searchCacheDb from './search-cache.js'; -import { ingest as collectVideoMetadata, syncListening, linkListening, startThumbnails } from './video-catalog.js'; +import { ingest as collectVideoMetadata, syncListening, linkListening, startThumbnails, backfillStep } from './video-catalog.js'; import { registerCatalogRoutes } from './recommendations.js'; import QRCode from 'qrcode'; import { createYtdlpPool } from './ytdlp-pool.js'; @@ -2319,7 +2319,7 @@ const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads, ingest: collectVideoMetadata, putSearch: searchCacheDb.put }); registerAdminToolsRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, media, - backupDir: process.env.UPLOAD_BACKUP_DIR || pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'backups') }); + backupDir: process.env.UPLOAD_BACKUP_DIR || pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'backups'), backfillStep }); // ============================================================================ // Uploads — the server's own searchable media library (see uploads.js) diff --git a/tests/admin-index.spec.js b/tests/admin-index.spec.js new file mode 100644 index 0000000..cfcb6b0 --- /dev/null +++ b/tests/admin-index.spec.js @@ -0,0 +1,22 @@ +const { test, expect } = require('@playwright/test'); + +test('search index tab shows source coverage and runs bounded repair actions', async ({ page }) => { + const actions = []; + await page.route('**/api/**', async route => { + const req = route.request(), url = new URL(req.url()); let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { enabled: true, admin: true }; + else if (url.pathname === '/api/admin/index/health') body = { ok: true, total: 100, urlCount: 90, detailCount: 40, thumbnailCount: 75, urlCoverage: .9, detailCoverage: .4, thumbnailCoverage: .75, sources: [{ source: 'deep', videos: 70, discoveries: 300, lastSeen: 1791900000000 }], lastRun: { status: 'complete', createdAt: 1791900000000, progress: { queriesDone: 20, newVideos: 400 } }, backfill: [{ source: 'media_cache', cursor: 10, ceiling: 20, done: false }] }; + else if (url.pathname === '/api/admin/index/backfill' && req.method() === 'POST') { actions.push('backfill'); body.processed = 3; } + else if (url.pathname === '/api/admin/index/cleanup' && req.method() === 'POST') { actions.push('cleanup'); body.removed = 2; } + else if (url.pathname === '/api/admin/media') body.media = []; + else if (url.pathname === '/api/uploads') body.uploads = []; + else if (url.pathname === '/api/admin/flags') body.flags = []; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.goto('/admin.html#index'); + await expect(page.locator('#indexMetrics')).toContainText('90%'); + await expect(page.locator('#indexSources')).toContainText('deep'); + await page.locator('#indexBackfill').click(); await expect(page.locator('#indexStatus')).toContainText('Processed 3'); + page.once('dialog', dialog => dialog.accept()); await page.locator('#indexCleanup').click(); await expect(page.locator('#indexStatus')).toContainText('Removed 2'); + expect(actions).toEqual(['backfill', 'cleanup']); +});