diff --git a/server/media-cache.js b/server/media-cache.js index 0496912..14bc7cb 100644 --- a/server/media-cache.js +++ b/server/media-cache.js @@ -158,7 +158,7 @@ function metaFromInfo(info, duration) { // getInfo async (videoId) => yt-dlp -J info — duration, live flags, meta // ---------------------------------------------------------------------------- export function createMediaCache({ - dir, db, download, getInfo, + dir, db, download, getInfo, getSourceUrl = id => `https://www.youtube.com/watch?v=${id}`, ffmpeg = 'ffmpeg', ffprobe = 'ffprobe', nice = 'nice', maxBytes = 10 * 1024 ** 3, minFreeBytes = 5 * 1024 ** 3, @@ -314,7 +314,9 @@ export function createMediaCache({ await job.persisted; try { await setStatus(job, 'downloading'); - const info = await getInfo(id); + const sourceUrl = await getSourceUrl(id); + if (!sourceUrl || !/^https:\/\/www\.youtube\.com\/watch\?v=[\w-]{11}$/.test(sourceUrl)) throw new MediaSkip('source URL is unavailable'); + const info = await getInfo(id, sourceUrl); const live = info.is_live || ['is_live', 'post_live', 'is_upcoming'].includes(info.live_status); if (live) throw new MediaSkip('live streams are not cached'); const expected = typeof info.duration === 'number' ? info.duration : 0; @@ -324,7 +326,7 @@ export function createMediaCache({ } await makeRoom(id, Math.max(expected, 60) * EST_BYTES_PER_SEC); - const raw = await download(id, `${base}.dl.mp4`); + const raw = await download(id, `${base}.dl.mp4`, sourceUrl); await setStatus(job, 'validating'); // Normalise: first video + first audio, moov atom up front so the copy diff --git a/server/media-cache.test.js b/server/media-cache.test.js index c44ffc4..a422c06 100644 --- a/server/media-cache.test.js +++ b/server/media-cache.test.js @@ -2,7 +2,7 @@ // Uses real ffmpeg fixtures and the real db.js against a throwaway DB file. import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { spawnSync } from 'node:child_process'; -import { mkdtempSync, rmSync, copyFileSync, writeFileSync, readFileSync, existsSync, readdirSync } from 'node:fs'; +import { mkdtempSync, rmSync, copyFileSync, writeFileSync, readFileSync, existsSync, readdirSync, unlinkSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -93,6 +93,7 @@ let dirN = 0; function makeCache({ fixture = 'good.mp4', duration = 10, ...opts } = {}) { const dir = join(root, 'media' + dirN++); const calls = []; + const sources = []; const db = { getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia, listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia, @@ -102,13 +103,13 @@ function makeCache({ fixture = 'good.mp4', duration = 10, ...opts } = {}) { const cache = createMediaCache({ dir, db, log: silentLog, getInfo: async (id) => ({ id, title: 'T ' + id, channel: 'C', duration }), - download: async (id, out) => { calls.push(id); await sleep(30); copyFileSync(fx(state.fixture), out); return out; }, + download: async (id, out, sourceUrl) => { calls.push(id); sources.push({ id, sourceUrl }); await sleep(30); copyFileSync(fx(state.fixture), out); return out; }, transcode: { enabled: false }, freeBytes: () => 100 * 1024 ** 3, backfillDelayMs: -1, ...opts, }); - return { cache, dir, calls, state }; + return { cache, dir, calls, sources, state }; } async function clearDb() { for (const r of await dbmod.listMedia()) await dbmod.deleteMedia(r.video_id); @@ -233,6 +234,19 @@ describe('media cache jobs', () => { expect(calls.length).toBe(2); }); + test('missing cached files are re-fetched from the stored source URL', async () => { + await clearDb(); + const id = 'srcAAAAAA01'; + const sourceUrl = `https://www.youtube.com/watch?v=${id}`; + const { cache, dir, calls, sources } = makeCache({ getSourceUrl: async () => sourceUrl }); + await cache.init(); + const first = await cache.ensureCached(id, { priority: HIGH }); + unlinkSync(join(dir, `${id}.${first.gen}.mp4`)); + await cache.ensureCached(id, { priority: HIGH }); + expect(calls).toHaveLength(2); + expect(sources.every(call => call.id === id && call.sourceUrl === sourceUrl)).toBe(true); + }); + test('auto jobs respect the auto length limit; explicit saves do not', async () => { await clearDb(); const { cache } = makeCache({ duration: 7200, autoMaxSeconds: 3600 }); diff --git a/server/notes.js b/server/notes.js index 88e1fbb..bf68373 100644 --- a/server/notes.js +++ b/server/notes.js @@ -314,6 +314,7 @@ export function verifyAdminCookie(secret, value, nowSec = Math.floor(Date.now() export function registerNoteRoutes(app, deps) { const { db, getProfile, profileNameRe, runYtdlp, adminPassword, backupDir, adminHtmlPath, workerToken, + catalog = null, media = null, } = deps; const ADMIN_COOKIE = 'ytp_admin'; const ADMIN_TTL = 30 * 24 * 3600; @@ -695,6 +696,7 @@ export function registerNoteRoutes(app, deps) { const n = await db.getNotes(r.video_id); out.push({ id: r.video_id, cacheStatus: r.status, title: meta.title || '', channel: meta.channel || meta.uploader || '', + sourceUrl: catalog ? await catalog.getSourceUrl(r.video_id) : null, duration: Number(r.duration) || 0, lastAccess: Number(r.last_access) || 0, lyricsRev: n.lyrics ? n.lyrics.rev : 0, lyricsLines: n.lyrics ? n.lyrics.data.lines.length : 0, }); @@ -703,6 +705,12 @@ export function registerNoteRoutes(app, deps) { return c.json({ ok: true, media: out }); }); + app.post('/api/admin/media/:id/redownload', requireAdminOrToken, async c => { + if (!media) return c.json({ ok: false, error: 'media cache unavailable' }, 503); + try { return c.json({ ok: true, ...(await media.redownload(c.req.param('id'))) }, 202); } + catch (err) { return c.json({ ok: false, error: err.message }, 400); } + }); + app.get('/api/admin/notes/recent', requireAdmin, async (c) => c.json({ ok: true, revs: await db.recentNoteRevs(150) })); app.post('/api/admin/notes/:id/:kind/restore', requireAdmin, async (c) => { diff --git a/server/notes.test.js b/server/notes.test.js index cd2a66f..65a9d7a 100644 --- a/server/notes.test.js +++ b/server/notes.test.js @@ -57,6 +57,8 @@ beforeAll(async () => { backupDir: join(root, 'backups'), adminHtmlPath: htmlPath, workerToken: 'w'.repeat(32), + catalog: { getSourceUrl: async id => /^[\w-]{11}$/.test(id) ? `https://www.youtube.com/watch?v=${id}` : null }, + media: { redownload: async id => ({ status: `queued:${id}` }) }, }); }); @@ -278,5 +280,7 @@ test('admin song list includes metadata-only phone songs for lyrics lookup', asy const cookie = await adminCookie(); const res = await app.request('/api/admin/media', { headers: { Cookie: cookie } }); const song = (await res.json()).media.find(x => x.id === id); - expect(song).toMatchObject({ title: 'Phone song', channel: 'Phone artist', duration: 240, cacheStatus: 'failed' }); + expect(song).toMatchObject({ title: 'Phone song', channel: 'Phone artist', duration: 240, cacheStatus: 'failed', sourceUrl: 'https://www.youtube.com/watch?v=phoneSong01' }); + expect((await app.request(`/api/admin/media/${id}/redownload`, { method: 'POST', headers: { Cookie: cookie } })).status).toBe(202); + expect((await app.request('/api/admin/media/phoneSong01/redownload', { method: 'POST' })).status).toBe(401); }); diff --git a/server/server.js b/server/server.js index 7989f36..ec72915 100644 --- a/server/server.js +++ b/server/server.js @@ -1104,7 +1104,7 @@ const SAVE_DEFAULT_FORMAT = // again) share ONE yt-dlp run instead of spawning another each time. const inflightDownloads = new Map(); // key -> { promise, waiters, tmpBase } -async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) { +async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal, sourceUrl = null) { await assertNotLive(videoId); const key = videoId + '|' + formatArgs.join(' '); let entry = inflightDownloads.get(key); @@ -1116,7 +1116,7 @@ async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) { entry.promise = withSaveSlot(() => { if (ctl.signal.aborted) throw new Error('save cancelled'); return runYtdlpResilient([ - `https://www.youtube.com/watch?v=${videoId}`, + sourceUrl || await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`, '--no-warnings', '--no-playlist', ...formatArgs, '--limit-rate', DOWNLOAD_RATE, @@ -1187,7 +1187,7 @@ async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal, cachedSrc // 1) Grab the full source (video+audio merged) so ffmpeg has both streams // — unless the server cache already holds it. if (!cachedSrc) await withSaveSlot(() => runYtdlpResilient([ - `https://www.youtube.com/watch?v=${videoId}`, + await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`, '--no-warnings', '--no-playlist', '-f', SAVE_MUX_FORMAT, '--merge-output-format', 'mp4', @@ -1243,6 +1243,7 @@ if (!MEDIA_VOLUME_MARKER || existsSync(MEDIA_VOLUME_MARKER)) mkdirSync(MEDIA_DIR const media = createMediaCache({ dir: MEDIA_DIR, volumeMarker: MEDIA_VOLUME_MARKER, + getSourceUrl: videoCatalog.getSourceUrl, db: { getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats, // Retention (docs/p2p-architecture.md flow 9): cold copies go before popular ones. @@ -1251,9 +1252,9 @@ const media = createMediaCache({ }), }, getInfo: async (videoId) => (await resolveStreams(videoId)).info, - download: async (videoId, out) => { + download: async (videoId, out, sourceUrl) => { await withSaveSlot(() => runYtdlpResilient([ - `https://www.youtube.com/watch?v=${videoId}`, + sourceUrl || `https://www.youtube.com/watch?v=${videoId}`, '--no-warnings', '--no-playlist', '-f', SAVE_MUX_FORMAT, '--merge-output-format', 'mp4', diff --git a/server/video-catalog.js b/server/video-catalog.js index 902eceb..4f32acf 100644 --- a/server/video-catalog.js +++ b/server/video-catalog.js @@ -10,6 +10,11 @@ const canonicalThumb = id => `https://i.ytimg.com/vi/${id}/hqdefault.jpg`; const canonicalUrl = id => `https://www.youtube.com/watch?v=${id}`; export const foldSearchText = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase(); export const channelKey = c => String(c.channelId || c.channel || '').toLowerCase().replace(/\s*-\s*topic$|vevo$|\s+official$/g, '').trim(); +export async function getSourceUrl(id) { + if (!VIDEO_ID.test(String(id || ''))) return null; + const row = (await db.execute({ sql: 'SELECT url FROM video_meta WHERE id=?', args: [id] })).rows[0]; + return row?.url || `https://www.youtube.com/watch?v=${id}`; +} function safeThumbnail(url) { try { const u = new URL(url); return u.protocol === 'https:' && /^(?:i|i\d)\.ytimg\.com$/.test(u.hostname) && !u.port && !u.username && !u.password; } catch { return false; }