josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refetch missing media from catalog source

Browse Source
This commit is contained in:
Jonathan Sykes
2026-10-14 01:26:25 +08:00
parent 593ecaecd9
commit ca10fe6486
6 changed files with 46 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
server/media-cache.js
View File

@@ -158,7 +158,7 @@ function metaFromInfo(info, duration) {
// getInfo async (videoId) => yt-dlp -J info — duration, live flags, meta
// ----------------------------------------------------------------------------
export function createMediaCache({
dir, db, download, getInfo,
dir, db, download, getInfo, getSourceUrl = id => `https://www.youtube.com/watch?v=${id}`,
ffmpeg = 'ffmpeg', ffprobe = 'ffprobe', nice = 'nice',
maxBytes = 10 * 1024 ** 3,
minFreeBytes = 5 * 1024 ** 3,
@@ -314,7 +314,9 @@ export function createMediaCache({
await job.persisted;
try {
await setStatus(job, 'downloading');
const info = await getInfo(id);
const sourceUrl = await getSourceUrl(id);
if (!sourceUrl || !/^https:\/\/www\.youtube\.com\/watch\?v=[\w-]{11}$/.test(sourceUrl)) throw new MediaSkip('source URL is unavailable');
const info = await getInfo(id, sourceUrl);
const live = info.is_live || ['is_live', 'post_live', 'is_upcoming'].includes(info.live_status);
if (live) throw new MediaSkip('live streams are not cached');
const expected = typeof info.duration === 'number' ? info.duration : 0;
@@ -324,7 +326,7 @@ export function createMediaCache({
}
await makeRoom(id, Math.max(expected, 60) * EST_BYTES_PER_SEC);
const raw = await download(id, `${base}.dl.mp4`);
const raw = await download(id, `${base}.dl.mp4`, sourceUrl);
await setStatus(job, 'validating');
// Normalise: first video + first audio, moov atom up front so the copy

20
server/media-cache.test.js
View File

@@ -2,7 +2,7 @@
// Uses real ffmpeg fixtures and the real db.js against a throwaway DB file.
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, rmSync, copyFileSync, writeFileSync, readFileSync, existsSync, readdirSync } from 'node:fs';
import { mkdtempSync, rmSync, copyFileSync, writeFileSync, readFileSync, existsSync, readdirSync, unlinkSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
@@ -93,6 +93,7 @@ let dirN = 0;
function makeCache({ fixture = 'good.mp4', duration = 10, ...opts } = {}) {
const dir = join(root, 'media' + dirN++);
const calls = [];
const sources = [];
const db = {
getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia,
listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia,
@@ -102,13 +103,13 @@ function makeCache({ fixture = 'good.mp4', duration = 10, ...opts } = {}) {
const cache = createMediaCache({
dir, db, log: silentLog,
getInfo: async (id) => ({ id, title: 'T ' + id, channel: 'C', duration }),
download: async (id, out) => { calls.push(id); await sleep(30); copyFileSync(fx(state.fixture), out); return out; },
download: async (id, out, sourceUrl) => { calls.push(id); sources.push({ id, sourceUrl }); await sleep(30); copyFileSync(fx(state.fixture), out); return out; },
transcode: { enabled: false },
freeBytes: () => 100 * 1024 ** 3,
backfillDelayMs: -1,
...opts,
});
return { cache, dir, calls, state };
return { cache, dir, calls, sources, state };
}
async function clearDb() {
for (const r of await dbmod.listMedia()) await dbmod.deleteMedia(r.video_id);
@@ -233,6 +234,19 @@ describe('media cache jobs', () => {
expect(calls.length).toBe(2);
});
test('missing cached files are re-fetched from the stored source URL', async () => {
await clearDb();
const id = 'srcAAAAAA01';
const sourceUrl = `https://www.youtube.com/watch?v=${id}`;
const { cache, dir, calls, sources } = makeCache({ getSourceUrl: async () => sourceUrl });
await cache.init();
const first = await cache.ensureCached(id, { priority: HIGH });
unlinkSync(join(dir, `${id}.${first.gen}.mp4`));
await cache.ensureCached(id, { priority: HIGH });
expect(calls).toHaveLength(2);
expect(sources.every(call => call.id === id && call.sourceUrl === sourceUrl)).toBe(true);
});
test('auto jobs respect the auto length limit; explicit saves do not', async () => {
await clearDb();
const { cache } = makeCache({ duration: 7200, autoMaxSeconds: 3600 });

8
server/notes.js
View File

@@ -314,6 +314,7 @@ export function verifyAdminCookie(secret, value, nowSec = Math.floor(Date.now()
export function registerNoteRoutes(app, deps) {
const {
db, getProfile, profileNameRe, runYtdlp, adminPassword, backupDir, adminHtmlPath, workerToken,
catalog = null, media = null,
} = deps;
const ADMIN_COOKIE = 'ytp_admin';
const ADMIN_TTL = 30 * 24 * 3600;
@@ -695,6 +696,7 @@ export function registerNoteRoutes(app, deps) {
const n = await db.getNotes(r.video_id);
out.push({
id: r.video_id, cacheStatus: r.status, title: meta.title || '', channel: meta.channel || meta.uploader || '',
sourceUrl: catalog ? await catalog.getSourceUrl(r.video_id) : null,
duration: Number(r.duration) || 0, lastAccess: Number(r.last_access) || 0,
lyricsRev: n.lyrics ? n.lyrics.rev : 0, lyricsLines: n.lyrics ? n.lyrics.data.lines.length : 0,
});
@@ -703,6 +705,12 @@ export function registerNoteRoutes(app, deps) {
return c.json({ ok: true, media: out });
});
app.post('/api/admin/media/:id/redownload', requireAdminOrToken, async c => {
if (!media) return c.json({ ok: false, error: 'media cache unavailable' }, 503);
try { return c.json({ ok: true, ...(await media.redownload(c.req.param('id'))) }, 202); }
catch (err) { return c.json({ ok: false, error: err.message }, 400); }
});
app.get('/api/admin/notes/recent', requireAdmin, async (c) => c.json({ ok: true, revs: await db.recentNoteRevs(150) }));
app.post('/api/admin/notes/:id/:kind/restore', requireAdmin, async (c) => {

6
server/notes.test.js
View File

@@ -57,6 +57,8 @@ beforeAll(async () => {
backupDir: join(root, 'backups'),
adminHtmlPath: htmlPath,
workerToken: 'w'.repeat(32),
catalog: { getSourceUrl: async id => /^[\w-]{11}$/.test(id) ? `https://www.youtube.com/watch?v=${id}` : null },
media: { redownload: async id => ({ status: `queued:${id}` }) },
});
});
@@ -278,5 +280,7 @@ test('admin song list includes metadata-only phone songs for lyrics lookup', asy
const cookie = await adminCookie();
const res = await app.request('/api/admin/media', { headers: { Cookie: cookie } });
const song = (await res.json()).media.find(x => x.id === id);
expect(song).toMatchObject({ title: 'Phone song', channel: 'Phone artist', duration: 240, cacheStatus: 'failed' });
expect(song).toMatchObject({ title: 'Phone song', channel: 'Phone artist', duration: 240, cacheStatus: 'failed', sourceUrl: 'https://www.youtube.com/watch?v=phoneSong01' });
expect((await app.request(`/api/admin/media/${id}/redownload`, { method: 'POST', headers: { Cookie: cookie } })).status).toBe(202);
expect((await app.request('/api/admin/media/phoneSong01/redownload', { method: 'POST' })).status).toBe(401);
});

11
server/server.js
View File

@@ -1104,7 +1104,7 @@ const SAVE_DEFAULT_FORMAT =
// again) share ONE yt-dlp run instead of spawning another each time.
const inflightDownloads = new Map(); // key -> { promise, waiters, tmpBase }
async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) {
async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal, sourceUrl = null) {
await assertNotLive(videoId);
const key = videoId + '|' + formatArgs.join(' ');
let entry = inflightDownloads.get(key);
@@ -1116,7 +1116,7 @@ async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) {
entry.promise = withSaveSlot(() => {
if (ctl.signal.aborted) throw new Error('save cancelled');
return runYtdlpResilient([
`https://www.youtube.com/watch?v=${videoId}`,
sourceUrl || await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`,
'--no-warnings', '--no-playlist',
...formatArgs,
'--limit-rate', DOWNLOAD_RATE,
@@ -1187,7 +1187,7 @@ async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal, cachedSrc
// 1) Grab the full source (video+audio merged) so ffmpeg has both streams
// — unless the server cache already holds it.
if (!cachedSrc) await withSaveSlot(() => runYtdlpResilient([
`https://www.youtube.com/watch?v=${videoId}`,
await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`,
'--no-warnings', '--no-playlist',
'-f', SAVE_MUX_FORMAT,
'--merge-output-format', 'mp4',
@@ -1243,6 +1243,7 @@ if (!MEDIA_VOLUME_MARKER || existsSync(MEDIA_VOLUME_MARKER)) mkdirSync(MEDIA_DIR
const media = createMediaCache({
dir: MEDIA_DIR,
volumeMarker: MEDIA_VOLUME_MARKER,
getSourceUrl: videoCatalog.getSourceUrl,
db: {
getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats,
// Retention (docs/p2p-architecture.md flow 9): cold copies go before popular ones.
@@ -1251,9 +1252,9 @@ const media = createMediaCache({
}),
},
getInfo: async (videoId) => (await resolveStreams(videoId)).info,
download: async (videoId, out) => {
download: async (videoId, out, sourceUrl) => {
await withSaveSlot(() => runYtdlpResilient([
`https://www.youtube.com/watch?v=${videoId}`,
sourceUrl || `https://www.youtube.com/watch?v=${videoId}`,
'--no-warnings', '--no-playlist',
'-f', SAVE_MUX_FORMAT,
'--merge-output-format', 'mp4',

5
server/video-catalog.js
View File

@@ -10,6 +10,11 @@ const canonicalThumb = id => `https://i.ytimg.com/vi/${id}/hqdefault.jpg`;
const canonicalUrl = id => `https://www.youtube.com/watch?v=${id}`;
export const foldSearchText = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
export const channelKey = c => String(c.channelId || c.channel || '').toLowerCase().replace(/\s*-\s*topic$|vevo$|\s+official$/g, '').trim();
export async function getSourceUrl(id) {
if (!VIDEO_ID.test(String(id || ''))) return null;
const row = (await db.execute({ sql: 'SELECT url FROM video_meta WHERE id=?', args: [id] })).rows[0];
return row?.url || `https://www.youtube.com/watch?v=${id}`;
}
function safeThumbnail(url) {
try { const u = new URL(url); return u.protocol === 'https:' && /^(?:i|i\d)\.ytimg\.com$/.test(u.hostname) && !u.port && !u.username && !u.password; }
catch { return false; }