josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add searchable index health and repair tools

Browse Source
This commit is contained in:
Jonathan Sykes
2026-10-14 02:02:07 +08:00
parent 42c68609ec
commit 34b1173ea6
6 changed files with 146 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
server/admin-tools.js
View File

@@ -4,7 +4,7 @@ const clientOf = db => typeof db?.execute === 'function' ? db : db?.db;
const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } };
const number = value => Number(value) || 0;
export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null }) {
export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null, backfillStep = null }) {
const sql = clientOf(db);
app.get('/api/admin/lyrics/quality', adminAuth, async c => {
@@ -97,4 +97,53 @@ export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, bac
return c.json({ ok: true, pinned: body.pinned });
});
app.get('/api/admin/index/health', adminAuth, async c => {
const [stats, sources, latest, backfill] = await Promise.all([
sql.execute(`SELECT COUNT(*) AS total,SUM(CASE WHEN url IS NOT NULL AND url<>'' THEN 1 ELSE 0 END) AS urls,
SUM(CASE WHEN EXISTS(SELECT 1 FROM video_details d WHERE d.video_id=video_meta.id) THEN 1 ELSE 0 END) AS details,
SUM(CASE WHEN EXISTS(SELECT 1 FROM video_thumbnails t WHERE t.video_id=video_meta.id AND t.data IS NOT NULL) THEN 1 ELSE 0 END) AS thumbnails FROM video_meta`),
sql.execute('SELECT source,COUNT(*) AS videos,SUM(discoveries) AS discoveries,MAX(last_seen) AS lastSeen FROM video_meta_sources GROUP BY source ORDER BY videos DESC'),
sql.execute('SELECT id,status,created_at,updated_at,state,error FROM deep_search_jobs ORDER BY created_at DESC LIMIT 1'),
sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source'),
]);
const row = stats.rows[0] || {}, total = number(row.total);
const last = latest.rows[0];
return c.json({ ok: true, total, urlCount: number(row.urls), detailCount: number(row.details), thumbnailCount: number(row.thumbnails),
urlCoverage: total ? number(row.urls) / total : 0, detailCoverage: total ? number(row.details) / total : 0, thumbnailCoverage: total ? number(row.thumbnails) / total : 0,
sources: sources.rows.map(x => ({ source: x.source, videos: number(x.videos), discoveries: number(x.discoveries), lastSeen: number(x.lastSeen) })),
lastRun: last ? { id: last.id, status: last.status, createdAt: number(last.created_at), updatedAt: number(last.updated_at), error: last.error || null, progress: parse(last.state) } : null,
backfill: backfill.rows.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) }, 200, { 'Cache-Control': 'no-store' });
});
app.post('/api/admin/index/backfill', adminAuth, async c => {
if (!backfillStep) return c.json({ ok: false, error: 'Catalog backfill is unavailable.' }, 503);
const processed = await backfillStep(100);
const state = (await sql.execute('SELECT source,cursor,ceiling FROM catalog_backfill ORDER BY source')).rows;
await audit(db, 'admin', 'index.backfill', '', { processed });
return c.json({ ok: true, processed, backfill: state.map(x => ({ source: x.source, cursor: number(x.cursor), ceiling: number(x.ceiling), done: number(x.cursor) >= number(x.ceiling) })) });
});
app.post('/api/admin/index/cleanup', adminAuth, async c => {
let body = {}; try { body = await c.req.json(); } catch {}
const days = Math.max(180, Math.min(3650, Number(body.days) || 365));
const cutoff = Date.now() - days * 86400_000;
const candidates = await sql.execute({ sql: `SELECT COUNT(*) AS n FROM video_meta m WHERE m.updated_at<?
AND NOT EXISTS(SELECT 1 FROM video_meta_sources s WHERE s.video_id=m.id)
AND NOT EXISTS(SELECT 1 FROM media_cache c WHERE c.video_id=m.id)
AND NOT EXISTS(SELECT 1 FROM listening_daily l WHERE l.video_id=m.id)`, args: [cutoff] });
const removed = number(candidates.rows[0]?.n);
await sql.execute({ sql: `DELETE FROM video_meta WHERE updated_at<?
AND NOT EXISTS(SELECT 1 FROM video_meta_sources s WHERE s.video_id=video_meta.id)
AND NOT EXISTS(SELECT 1 FROM media_cache c WHERE c.video_id=video_meta.id)
AND NOT EXISTS(SELECT 1 FROM listening_daily l WHERE l.video_id=video_meta.id)`, args: [cutoff] });
await sql.batch([
'DELETE FROM video_meta_sources WHERE video_id NOT IN (SELECT id FROM video_meta)',
'DELETE FROM video_thumbnails WHERE video_id NOT IN (SELECT id FROM video_meta)',
'DELETE FROM video_channels WHERE video_id NOT IN (SELECT id FROM video_meta)',
'DELETE FROM video_details WHERE video_id NOT IN (SELECT id FROM video_meta)',
], 'write');
await audit(db, 'admin', 'index.cleanup', '', { days, removed });
return c.json({ ok: true, removed, days });
});
}

30
server/admin-tools.test.js
View File

@@ -14,11 +14,17 @@ const auth = async (c, next) => c.req.header('x-admin') === 'yes' ? next() : c.j
beforeAll(async () => {
await db.initDb();
registerAdminToolsRoutes(app, { db, adminAuth: auth });
registerAdminToolsRoutes(app, { db, adminAuth: auth, backfillStep: async () => 3 });
await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] });
await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] });
await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] });
await db.db.execute({ sql: "INSERT INTO media_cache(video_id,status,size,gen) VALUES('CacheItem001','ready',700,1),('CacheItem002','ready',400,1)" });
await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at,url) VALUES(?,?,?,?,?,?)', args: ['IndexSong001', JSON.stringify({ title: 'Indexed song' }), 'indexed song', 1, Date.now(), 'https://www.youtube.com/watch?v=IndexSong001'] });
await db.db.execute({ sql: "INSERT INTO video_meta_sources(video_id,source,last_seen) VALUES('IndexSong001','deep',?)", args: [Date.now()] });
await db.db.execute({ sql: "INSERT INTO video_details(video_id,metadata,updated_at) VALUES('IndexSong001','{}',?)", args: [Date.now()] });
await db.db.execute({ sql: "INSERT INTO video_thumbnails(video_id,url,data,mime,size,fetched_at) VALUES('IndexSong001','https://i.ytimg.com/vi/IndexSong001/hqdefault.jpg',x'01','image/jpeg',1,?)", args: [Date.now()] });
await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,1)', args: ['OldOrphan01', JSON.stringify({ title: 'Old orphan' }), 'old orphan', 1] });
await db.db.execute({ sql: "INSERT INTO deep_search_jobs(id,status,plan,options,state,created_at,updated_at) VALUES('job-1','complete','[]','{}','{\"queriesDone\":2,\"newVideos\":5}',1,2)" });
});
afterAll(() => rmSync(root, { recursive: true, force: true }));
@@ -27,12 +33,12 @@ describe('lyrics quality queue', () => {
expect((await app.request('/api/admin/lyrics/quality')).status).toBe(401);
const headers = { 'x-admin': 'yes' };
const missing = await (await app.request('/api/admin/lyrics/quality?category=missing', { headers })).json();
expect(missing.total).toBe(1);
expect(missing.songs[0]).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] });
expect(missing.total).toBeGreaterThanOrEqual(1);
expect(missing.songs.find(song => song.id === 'QualMiss001')).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] });
const untimed = await (await app.request('/api/admin/lyrics/quality?category=untimed', { headers })).json();
expect(untimed.songs[0]).toMatchObject({ id: 'Untimely001', lines: 2, untimed: true });
const short = await (await app.request('/api/admin/lyrics/quality?category=short&minLines=3', { headers })).json();
expect(short.songs[0].reasons).toContain('short');
expect(short.songs.find(song => song.id === 'Untimely001').reasons).toContain('short');
const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json();
expect(page.songs).toHaveLength(1);
});
@@ -48,4 +54,20 @@ describe('lyrics quality queue', () => {
const health = await (await app.request('/api/admin/cache/health', { headers: { 'x-admin': 'yes' } })).json();
expect(health.pinnedBytes).toBe(700);
});
test('index health reports coverage, source and deep-search data; repair actions are audited', async () => {
const headers = { 'x-admin': 'yes', 'content-type': 'application/json' };
const health = await (await app.request('/api/admin/index/health', { headers })).json();
expect(health.total).toBeGreaterThanOrEqual(1);
expect(health.urlCount).toBeGreaterThanOrEqual(1);
expect(health.detailCount).toBe(1);
expect(health.thumbnailCount).toBe(1);
expect(health.sources[0].source).toBe('deep');
expect(health.lastRun.progress.queriesDone).toBe(2);
const step = await (await app.request('/api/admin/index/backfill', { method: 'POST', headers, body: '{}' })).json();
expect(step.processed).toBe(3);
const clean = await (await app.request('/api/admin/index/cleanup', { method: 'POST', headers, body: JSON.stringify({ days: 180 }) })).json();
expect(clean.removed).toBeGreaterThanOrEqual(1);
expect((await db.db.execute({ sql: "SELECT COUNT(*) AS n FROM video_meta WHERE id='OldOrphan01'" })).rows[0].n).toBe(0);
});
});

4
server/server.js
View File

@@ -64,7 +64,7 @@ import { sha256Range } from './hash.js';
import * as innertube from './innertube.js';
import { createRelatedService, registerRelatedRoute } from './related.js';
import * as searchCacheDb from './search-cache.js';
import { ingest as collectVideoMetadata, syncListening, linkListening, startThumbnails } from './video-catalog.js';
import { ingest as collectVideoMetadata, syncListening, linkListening, startThumbnails, backfillStep } from './video-catalog.js';
import { registerCatalogRoutes } from './recommendations.js';
import QRCode from 'qrcode';
import { createYtdlpPool } from './ytdlp-pool.js';
@@ -2319,7 +2319,7 @@ const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes
searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads,
ingest: collectVideoMetadata, putSearch: searchCacheDb.put });
registerAdminToolsRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, media,
backupDir: process.env.UPLOAD_BACKUP_DIR || pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'backups') });
backupDir: process.env.UPLOAD_BACKUP_DIR || pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'backups'), backfillStep });
// ============================================================================
// Uploads — the server's own searchable media library (see uploads.js)