Add searchable index health and repair tools

server/admin-tools.test.js

@@ -14,11 +14,17 @@ const auth = async (c, next) => c.req.header('x-admin') === 'yes' ? next() : c.j
 
 beforeAll(async () => {
   await db.initDb();
-  registerAdminToolsRoutes(app, { db, adminAuth: auth });
+  registerAdminToolsRoutes(app, { db, adminAuth: auth, backfillStep: async () => 3 });
   await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] });
   await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] });
   await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] });
   await db.db.execute({ sql: "INSERT INTO media_cache(video_id,status,size,gen) VALUES('CacheItem001','ready',700,1),('CacheItem002','ready',400,1)" });
+  await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at,url) VALUES(?,?,?,?,?,?)', args: ['IndexSong001', JSON.stringify({ title: 'Indexed song' }), 'indexed song', 1, Date.now(), 'https://www.youtube.com/watch?v=IndexSong001'] });
+  await db.db.execute({ sql: "INSERT INTO video_meta_sources(video_id,source,last_seen) VALUES('IndexSong001','deep',?)", args: [Date.now()] });
+  await db.db.execute({ sql: "INSERT INTO video_details(video_id,metadata,updated_at) VALUES('IndexSong001','{}',?)", args: [Date.now()] });
+  await db.db.execute({ sql: "INSERT INTO video_thumbnails(video_id,url,data,mime,size,fetched_at) VALUES('IndexSong001','https://i.ytimg.com/vi/IndexSong001/hqdefault.jpg',x'01','image/jpeg',1,?)", args: [Date.now()] });
+  await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,1)', args: ['OldOrphan01', JSON.stringify({ title: 'Old orphan' }), 'old orphan', 1] });
+  await db.db.execute({ sql: "INSERT INTO deep_search_jobs(id,status,plan,options,state,created_at,updated_at) VALUES('job-1','complete','[]','{}','{\"queriesDone\":2,\"newVideos\":5}',1,2)" });
 });
 afterAll(() => rmSync(root, { recursive: true, force: true }));
 
@@ -27,12 +33,12 @@ describe('lyrics quality queue', () => {
     expect((await app.request('/api/admin/lyrics/quality')).status).toBe(401);
     const headers = { 'x-admin': 'yes' };
     const missing = await (await app.request('/api/admin/lyrics/quality?category=missing', { headers })).json();
-    expect(missing.total).toBe(1);
-    expect(missing.songs[0]).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] });
+    expect(missing.total).toBeGreaterThanOrEqual(1);
+    expect(missing.songs.find(song => song.id === 'QualMiss001')).toMatchObject({ id: 'QualMiss001', reasons: ['missing'] });
     const untimed = await (await app.request('/api/admin/lyrics/quality?category=untimed', { headers })).json();
     expect(untimed.songs[0]).toMatchObject({ id: 'Untimely001', lines: 2, untimed: true });
     const short = await (await app.request('/api/admin/lyrics/quality?category=short&minLines=3', { headers })).json();
-    expect(short.songs[0].reasons).toContain('short');
+    expect(short.songs.find(song => song.id === 'Untimely001').reasons).toContain('short');
     const page = await (await app.request('/api/admin/lyrics/quality?limit=1&offset=1', { headers })).json();
     expect(page.songs).toHaveLength(1);
   });
@@ -48,4 +54,20 @@ describe('lyrics quality queue', () => {
     const health = await (await app.request('/api/admin/cache/health', { headers: { 'x-admin': 'yes' } })).json();
     expect(health.pinnedBytes).toBe(700);
   });
+
+  test('index health reports coverage, source and deep-search data; repair actions are audited', async () => {
+    const headers = { 'x-admin': 'yes', 'content-type': 'application/json' };
+    const health = await (await app.request('/api/admin/index/health', { headers })).json();
+    expect(health.total).toBeGreaterThanOrEqual(1);
+    expect(health.urlCount).toBeGreaterThanOrEqual(1);
+    expect(health.detailCount).toBe(1);
+    expect(health.thumbnailCount).toBe(1);
+    expect(health.sources[0].source).toBe('deep');
+    expect(health.lastRun.progress.queriesDone).toBe(2);
+    const step = await (await app.request('/api/admin/index/backfill', { method: 'POST', headers, body: '{}' })).json();
+    expect(step.processed).toBe(3);
+    const clean = await (await app.request('/api/admin/index/cleanup', { method: 'POST', headers, body: JSON.stringify({ days: 180 }) })).json();
+    expect(clean.removed).toBeGreaterThanOrEqual(1);
+    expect((await db.db.execute({ sql: "SELECT COUNT(*) AS n FROM video_meta WHERE id='OldOrphan01'" })).rows[0].n).toBe(0);
+  });
 });