Refetch missing media from catalog source

server/media-cache.js

@@ -158,7 +158,7 @@ function metaFromInfo(info, duration) {
 //   getInfo      async (videoId) => yt-dlp -J info  — duration, live flags, meta
 // ----------------------------------------------------------------------------
 export function createMediaCache({
-  dir, db, download, getInfo,
+  dir, db, download, getInfo, getSourceUrl = id => `https://www.youtube.com/watch?v=${id}`,
   ffmpeg = 'ffmpeg', ffprobe = 'ffprobe', nice = 'nice',
   maxBytes = 10 * 1024 ** 3,
   minFreeBytes = 5 * 1024 ** 3,
@@ -314,7 +314,9 @@ export function createMediaCache({
     await job.persisted;
     try {
       await setStatus(job, 'downloading');
-      const info = await getInfo(id);
+      const sourceUrl = await getSourceUrl(id);
+      if (!sourceUrl || !/^https:\/\/www\.youtube\.com\/watch\?v=[\w-]{11}$/.test(sourceUrl)) throw new MediaSkip('source URL is unavailable');
+      const info = await getInfo(id, sourceUrl);
       const live = info.is_live || ['is_live', 'post_live', 'is_upcoming'].includes(info.live_status);
       if (live) throw new MediaSkip('live streams are not cached');
       const expected = typeof info.duration === 'number' ? info.duration : 0;
@@ -324,7 +326,7 @@ export function createMediaCache({
       }
       await makeRoom(id, Math.max(expected, 60) * EST_BYTES_PER_SEC);
 
-      const raw = await download(id, `${base}.dl.mp4`);
+      const raw = await download(id, `${base}.dl.mp4`, sourceUrl);
       await setStatus(job, 'validating');
 
       // Normalise: first video + first audio, moov atom up front so the copy

server/media-cache.test.js

@@ -2,7 +2,7 @@
 // Uses real ffmpeg fixtures and the real db.js against a throwaway DB file.
 import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
 import { spawnSync } from 'node:child_process';
-import { mkdtempSync, rmSync, copyFileSync, writeFileSync, readFileSync, existsSync, readdirSync } from 'node:fs';
+import { mkdtempSync, rmSync, copyFileSync, writeFileSync, readFileSync, existsSync, readdirSync, unlinkSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 
@@ -93,6 +93,7 @@ let dirN = 0;
 function makeCache({ fixture = 'good.mp4', duration = 10, ...opts } = {}) {
   const dir = join(root, 'media' + dirN++);
   const calls = [];
+  const sources = [];
   const db = {
     getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia,
     listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia,
@@ -102,13 +103,13 @@ function makeCache({ fixture = 'good.mp4', duration = 10, ...opts } = {}) {
   const cache = createMediaCache({
     dir, db, log: silentLog,
     getInfo: async (id) => ({ id, title: 'T ' + id, channel: 'C', duration }),
-    download: async (id, out) => { calls.push(id); await sleep(30); copyFileSync(fx(state.fixture), out); return out; },
+    download: async (id, out, sourceUrl) => { calls.push(id); sources.push({ id, sourceUrl }); await sleep(30); copyFileSync(fx(state.fixture), out); return out; },
     transcode: { enabled: false },
     freeBytes: () => 100 * 1024 ** 3,
     backfillDelayMs: -1,
     ...opts,
   });
-  return { cache, dir, calls, state };
+  return { cache, dir, calls, sources, state };
 }
 async function clearDb() {
   for (const r of await dbmod.listMedia()) await dbmod.deleteMedia(r.video_id);
@@ -233,6 +234,19 @@ describe('media cache jobs', () => {
     expect(calls.length).toBe(2);
   });
 
+  test('missing cached files are re-fetched from the stored source URL', async () => {
+    await clearDb();
+    const id = 'srcAAAAAA01';
+    const sourceUrl = `https://www.youtube.com/watch?v=${id}`;
+    const { cache, dir, calls, sources } = makeCache({ getSourceUrl: async () => sourceUrl });
+    await cache.init();
+    const first = await cache.ensureCached(id, { priority: HIGH });
+    unlinkSync(join(dir, `${id}.${first.gen}.mp4`));
+    await cache.ensureCached(id, { priority: HIGH });
+    expect(calls).toHaveLength(2);
+    expect(sources.every(call => call.id === id && call.sourceUrl === sourceUrl)).toBe(true);
+  });
+
   test('auto jobs respect the auto length limit; explicit saves do not', async () => {
     await clearDb();
     const { cache } = makeCache({ duration: 7200, autoMaxSeconds: 3600 });

server/notes.js

@@ -314,6 +314,7 @@ export function verifyAdminCookie(secret, value, nowSec = Math.floor(Date.now()
 export function registerNoteRoutes(app, deps) {
   const {
     db, getProfile, profileNameRe, runYtdlp, adminPassword, backupDir, adminHtmlPath, workerToken,
+    catalog = null, media = null,
   } = deps;
   const ADMIN_COOKIE = 'ytp_admin';
   const ADMIN_TTL = 30 * 24 * 3600;
@@ -695,6 +696,7 @@ export function registerNoteRoutes(app, deps) {
       const n = await db.getNotes(r.video_id);
       out.push({
         id: r.video_id, cacheStatus: r.status, title: meta.title || '', channel: meta.channel || meta.uploader || '',
+        sourceUrl: catalog ? await catalog.getSourceUrl(r.video_id) : null,
         duration: Number(r.duration) || 0, lastAccess: Number(r.last_access) || 0,
         lyricsRev: n.lyrics ? n.lyrics.rev : 0, lyricsLines: n.lyrics ? n.lyrics.data.lines.length : 0,
       });
@@ -703,6 +705,12 @@ export function registerNoteRoutes(app, deps) {
     return c.json({ ok: true, media: out });
   });
 
+  app.post('/api/admin/media/:id/redownload', requireAdminOrToken, async c => {
+    if (!media) return c.json({ ok: false, error: 'media cache unavailable' }, 503);
+    try { return c.json({ ok: true, ...(await media.redownload(c.req.param('id'))) }, 202); }
+    catch (err) { return c.json({ ok: false, error: err.message }, 400); }
+  });
+
   app.get('/api/admin/notes/recent', requireAdmin, async (c) => c.json({ ok: true, revs: await db.recentNoteRevs(150) }));
 
   app.post('/api/admin/notes/:id/:kind/restore', requireAdmin, async (c) => {

server/notes.test.js

@@ -57,6 +57,8 @@ beforeAll(async () => {
     backupDir: join(root, 'backups'),
     adminHtmlPath: htmlPath,
     workerToken: 'w'.repeat(32),
+    catalog: { getSourceUrl: async id => /^[\w-]{11}$/.test(id) ? `https://www.youtube.com/watch?v=${id}` : null },
+    media: { redownload: async id => ({ status: `queued:${id}` }) },
   });
 });
 
@@ -278,5 +280,7 @@ test('admin song list includes metadata-only phone songs for lyrics lookup', asy
   const cookie = await adminCookie();
   const res = await app.request('/api/admin/media', { headers: { Cookie: cookie } });
   const song = (await res.json()).media.find(x => x.id === id);
-  expect(song).toMatchObject({ title: 'Phone song', channel: 'Phone artist', duration: 240, cacheStatus: 'failed' });
+  expect(song).toMatchObject({ title: 'Phone song', channel: 'Phone artist', duration: 240, cacheStatus: 'failed', sourceUrl: 'https://www.youtube.com/watch?v=phoneSong01' });
+  expect((await app.request(`/api/admin/media/${id}/redownload`, { method: 'POST', headers: { Cookie: cookie } })).status).toBe(202);
+  expect((await app.request('/api/admin/media/phoneSong01/redownload', { method: 'POST' })).status).toBe(401);
 });

server/server.js

@@ -1104,7 +1104,7 @@ const SAVE_DEFAULT_FORMAT =
 // again) share ONE yt-dlp run instead of spawning another each time.
 const inflightDownloads = new Map(); // key -> { promise, waiters, tmpBase }
 
-async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) {
+async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal, sourceUrl = null) {
   await assertNotLive(videoId);
   const key = videoId + '|' + formatArgs.join(' ');
   let entry = inflightDownloads.get(key);
@@ -1116,7 +1116,7 @@ async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) {
     entry.promise = withSaveSlot(() => {
       if (ctl.signal.aborted) throw new Error('save cancelled');
       return runYtdlpResilient([
-        `https://www.youtube.com/watch?v=${videoId}`,
+        sourceUrl || await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`,
         '--no-warnings', '--no-playlist',
         ...formatArgs,
         '--limit-rate', DOWNLOAD_RATE,
@@ -1187,7 +1187,7 @@ async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal, cachedSrc
     // 1) Grab the full source (video+audio merged) so ffmpeg has both streams
     //    — unless the server cache already holds it.
     if (!cachedSrc) await withSaveSlot(() => runYtdlpResilient([
-      `https://www.youtube.com/watch?v=${videoId}`,
+      await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`,
       '--no-warnings', '--no-playlist',
       '-f', SAVE_MUX_FORMAT,
       '--merge-output-format', 'mp4',
@@ -1243,6 +1243,7 @@ if (!MEDIA_VOLUME_MARKER || existsSync(MEDIA_VOLUME_MARKER)) mkdirSync(MEDIA_DIR
 const media = createMediaCache({
   dir: MEDIA_DIR,
   volumeMarker: MEDIA_VOLUME_MARKER,
+  getSourceUrl: videoCatalog.getSourceUrl,
   db: {
     getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats,
     // Retention (docs/p2p-architecture.md flow 9): cold copies go before popular ones.
@@ -1251,9 +1252,9 @@ const media = createMediaCache({
     }),
   },
   getInfo: async (videoId) => (await resolveStreams(videoId)).info,
-  download: async (videoId, out) => {
+  download: async (videoId, out, sourceUrl) => {
     await withSaveSlot(() => runYtdlpResilient([
-      `https://www.youtube.com/watch?v=${videoId}`,
+      sourceUrl || `https://www.youtube.com/watch?v=${videoId}`,
       '--no-warnings', '--no-playlist',
       '-f', SAVE_MUX_FORMAT,
       '--merge-output-format', 'mp4',

server/video-catalog.js

@@ -10,6 +10,11 @@ const canonicalThumb = id => `https://i.ytimg.com/vi/${id}/hqdefault.jpg`;
 const canonicalUrl = id => `https://www.youtube.com/watch?v=${id}`;
 export const foldSearchText = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
 export const channelKey = c => String(c.channelId || c.channel || '').toLowerCase().replace(/\s*-\s*topic$|vevo$|\s+official$/g, '').trim();
+export async function getSourceUrl(id) {
+  if (!VIDEO_ID.test(String(id || ''))) return null;
+  const row = (await db.execute({ sql: 'SELECT url FROM video_meta WHERE id=?', args: [id] })).rows[0];
+  return row?.url || `https://www.youtube.com/watch?v=${id}`;
+}
 function safeThumbnail(url) {
   try { const u = new URL(url); return u.protocol === 'https:' && /^(?:i|i\d)\.ytimg\.com$/.test(u.hostname) && !u.port && !u.username && !u.password; }
   catch { return false; }