/* ============================================================================ * media-cache.js — server-side video cache fed by background jobs * * Every video that is played or saved gets ONE validated copy on disk: * /..mp4 ≤720p H.264 (8-bit 4:2:0) + AAC, faststart * /..m4a audio-only sidecar (stream copy) for audio mode * * - Jobs are owned by the server, not by an HTTP request: a client closing * its tab never kills a download, the copy just lands for next time. * - Nothing is promoted without passing validateMedia() — a truncated or * undecodable file must never be served "forever". * - Copies never expire by time. They leave only through LRU eviction at the * byte budget, or through redownload() (the client's "Broken" button). * - `gen` bumps on every replacement and is part of the filename, so a URL * carrying ?g= always maps to the same bytes; superseded files are kept * for a grace period so in-flight Range playback never splices two encodes. * - After a copy is ready, a separate low-priority lane re-encodes it with * x264 CRF and keeps the result only when it is meaningfully smaller. * ========================================================================== */ import { spawn } from 'node:child_process'; import { copyFileSync, existsSync, linkSync, mkdirSync, readdirSync, renameSync, statSync, statfsSync, unlinkSync, } from 'node:fs'; import { join } from 'node:path'; export const HIGH = 0; // explicit save / Broken re-download export const LOW = 1; // auto-cache after a play const ID_RE = /^[A-Za-z0-9_-]{6,64}$/; export const isMediaId = (id) => typeof id === 'string' && ID_RE.test(id); const MIN_BYTES = 100 * 1024; const OLD_GEN_GRACE_MS = 30 * 60_000; // keep superseded files for in-flight playback const EVICT_PROTECT_MS = 10 * 60_000; // never evict something played this recently const TOUCH_DEBOUNCE_MS = 60_000; const EST_BYTES_PER_SEC = 250 * 1024; // ~2 Mbps: pre-compression 720p + 128k AAC const MIN_SAVING = 0.15; // keep an x264 re-encode only if ≥15% smaller const DAY_MS = 24 * 3600_000; // A condition that means "don't cache this right now, just stream it" — not a // broken download. code: SKIPPED (live, too long, disk/budget) | BACKOFF. export class MediaSkip extends Error { constructor(message, code = 'SKIPPED') { super(message); this.code = code; } } const tail = (s, n = 400) => String(s || '').trim().slice(-n); const sizeOf = (p) => { try { return statSync(p).size; } catch { return 0; } }; const fmtMB = (b) => (b / 1048576).toFixed(1) + ' MB'; // Spawn a process and collect its output. Never rejects. function run(cmd, args) { return new Promise((resolve) => { let child; try { child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'pipe'] }); } catch (e) { resolve({ code: -1, stdout: '', stderr: e.message }); return; } let out = ''; let err = ''; child.stdout.setEncoding('utf8'); child.stderr.setEncoding('utf8'); child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { err = (err + d).slice(-8000); }); child.on('error', (e) => resolve({ code: -1, stdout: out, stderr: e.message })); child.on('close', (code) => resolve({ code, stdout: out, stderr: err })); }); } // ---------------------------------------------------------------------------- // Validation gate. Throws with a reason; resolves to probe facts on success. // 1. file exists and is not tiny // 2. exactly one H.264 8-bit 4:2:0 video stream + an AAC audio stream // (AV1/VP9/10-bit play on almost nothing Apple — see SAVE_MUX_FORMAT) // 3. container AND per-stream durations cover the expected length // 4. full demux pass reports no errors (catches corrupt/truncated boxes) // 5. the first and last seconds actually decode // ---------------------------------------------------------------------------- export async function validateMedia(path, expected, { ffmpeg = 'ffmpeg', ffprobe = 'ffprobe' } = {}) { const size = sizeOf(path); if (!size) throw new Error('validation: output file missing'); if (size < MIN_BYTES) throw new Error(`validation: file too small (${size} bytes)`); const pr = await run(ffprobe, [ '-v', 'error', '-show_entries', 'format=duration:stream=codec_type,codec_name,height,pix_fmt,duration:stream_disposition=attached_pic', '-of', 'json', path, ]); if (pr.code !== 0) throw new Error('validation: ffprobe failed: ' + tail(pr.stderr)); let j; try { j = JSON.parse(pr.stdout); } catch { throw new Error('validation: unreadable ffprobe output'); } const streams = Array.isArray(j.streams) ? j.streams : []; const video = streams.filter((s) => s.codec_type === 'video' && !(s.disposition && s.disposition.attached_pic)); const audio = streams.filter((s) => s.codec_type === 'audio'); if (video.length !== 1) throw new Error(`validation: expected 1 video stream, found ${video.length}`); const v = video[0]; if (v.codec_name !== 'h264') throw new Error(`validation: video codec ${v.codec_name} is not h264`); if (v.pix_fmt && !/^yuvj?420p$/.test(v.pix_fmt)) throw new Error(`validation: pixel format ${v.pix_fmt} is not 8-bit 4:2:0`); const a = audio.find((s) => s.codec_name === 'aac'); if (!a) throw new Error('validation: no AAC audio stream'); const duration = Number(j.format && j.format.duration) || 0; if (duration <= 0) throw new Error('validation: zero duration'); if (expected > 0) { const min = expected - Math.max(3, expected * 0.03); for (const [label, d] of [['file', duration], ['video', Number(v.duration)], ['audio', Number(a.duration)]]) { if (Number.isFinite(d) && d > 0 && d < min) { throw new Error(`validation: ${label} truncated (${d.toFixed(1)}s of ${expected}s)`); } } } const demux = await run(ffmpeg, ['-v', 'error', '-nostdin', '-i', path, '-map', '0', '-c', 'copy', '-f', 'null', '-']); if (demux.code !== 0 || demux.stderr.trim()) throw new Error('validation: demux errors: ' + tail(demux.stderr)); const head = await run(ffmpeg, ['-v', 'error', '-nostdin', '-i', path, '-t', '3', '-f', 'null', '-']); if (head.code !== 0 || head.stderr.trim()) throw new Error('validation: head decode failed: ' + tail(head.stderr)); const end = await run(ffmpeg, ['-v', 'error', '-nostdin', '-sseof', '-5', '-i', path, '-f', 'null', '-']); if (end.code !== 0 || end.stderr.trim()) throw new Error('validation: tail decode failed: ' + tail(end.stderr)); return { size, height: v.height || 0, vcodec: v.codec_name, acodec: a.codec_name, duration }; } function metaFromInfo(info, duration) { const s = (...keys) => { for (const k of keys) if (typeof info[k] === 'string' && info[k].trim()) return info[k].trim(); return ''; }; return { title: s('title') || '(untitled)', channel: s('channel', 'uploader'), channelId: s('channel_id', 'uploader_id'), channelUrl: s('channel_url', 'uploader_url'), duration: typeof info.duration === 'number' ? info.duration : Math.round(duration || 0), }; } // ---------------------------------------------------------------------------- // createMediaCache(opts) // dir cache directory (tmp work dir lives in /.tmp, same fs) // db { getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats } // download async (videoId, outPath) => path — yt-dlp into outPath // getInfo async (videoId) => yt-dlp -J info — duration, live flags, meta // ---------------------------------------------------------------------------- export function createMediaCache({ dir, db, download, getInfo, ffmpeg = 'ffmpeg', ffprobe = 'ffprobe', nice = 'nice', maxBytes = 10 * 1024 ** 3, minFreeBytes = 5 * 1024 ** 3, autoMaxSeconds = 3600, saveMaxSeconds = 3 * 3600, transcode = { enabled: true, crf: 28, preset: 'slow', threads: 2 }, freeBytes = () => { const s = statfsSync(dir); return s.bavail * s.bsize; }, now = () => Date.now(), log = console, } = {}) { const tmpDir = join(dir, '.tmp'); const fileFor = (id, gen, kind = 'mp4') => join(dir, `${id}.${gen}.${kind}`); const jobs = new Map(); // videoId -> fetch job (queued or running) const fetchQueue = []; let fetchActive = null; // job currently downloading/validating let seq = 0; const optQueue = []; const optPending = new Set(); let optActive = null; // videoId currently being re-encoded const lastTouch = new Map(); function sweepTmp(prefix) { let names = []; try { names = readdirSync(tmpDir); } catch { return; } for (const n of names) { if (!prefix || n.startsWith(prefix)) { try { unlinkSync(join(tmpDir, n)); } catch { /* gone */ } } } } // Delete this video's files, optionally keeping one generation. Superseded // generations can be deleted after a delay so playback in flight finishes. function removeFiles(id, keepGen = null, delayMs = 0) { let names = []; try { names = readdirSync(dir); } catch { return; } for (const n of names) { const m = /^(.+)\.(\d+)\.(mp4|m4a)$/.exec(n); if (!m || m[1] !== id || (keepGen !== null && Number(m[2]) === keepGen)) continue; const p = join(dir, n); const rm = () => { try { unlinkSync(p); } catch { /* gone */ } }; if (delayMs > 0) { const t = setTimeout(rm, delayMs); if (t && t.unref) t.unref(); } else rm(); } } function readyFilesExist(row) { return existsSync(fileFor(row.video_id, row.gen)) && existsSync(fileFor(row.video_id, row.gen, 'm4a')); } function safeFree() { try { return freeBytes(); } catch { return null; } } async function evict(id, why) { await db.deleteMedia(id); removeFiles(id); log.info?.(`[media] evicted ${id} (${why})`); } // Make room for ~est bytes under the budget (LRU), then check the disk guard. async function makeRoom(id, est) { let { bytes } = await db.mediaStats(); if (bytes + est > maxBytes) { const cutoff = now() - EVICT_PROTECT_MS; for (const r of await db.listMediaLru()) { if (bytes + est <= maxBytes) break; if (r.video_id === id || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue; await evict(r.video_id, 'budget'); bytes -= r.size || 0; } } if (bytes + est > maxBytes) throw new MediaSkip('cache budget is full of recently played videos'); const free = safeFree(); if (free !== null && free - est < minFreeBytes) { throw new MediaSkip(`disk guard: only ${fmtMB(free)} free on the cache volume`); } } async function setStatus(job, status) { job.status = status; await db.upsertMedia(job.id, { status, updated_at: now() }); } function bump(job, priority, auto) { if (priority < job.priority) job.priority = priority; if (!auto) job.auto = false; // an explicit request lifts the auto length limit } function startJob(id, priority, auto, row) { let resolve, reject; const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); promise.catch(() => {}); // auto jobs have nobody awaiting them const job = { id, priority, auto, seq: seq++, status: 'queued', promise, resolve, reject }; job.persisted = db.upsertMedia(id, { status: 'queued', priority, auto: auto ? 1 : 0, gen: (row && row.gen) || 0, created_at: (row && row.created_at) || now(), updated_at: now(), }).catch((e) => log.warn?.(`[media] ${id} queue write failed: ${e.message}`)); jobs.set(id, job); fetchQueue.push(job); pumpFetch(); return promise; } function pumpFetch() { if (fetchActive || !fetchQueue.length) return; fetchQueue.sort((a, b) => a.priority - b.priority || a.seq - b.seq); const job = fetchQueue.shift(); fetchActive = job; runFetch(job).then(job.resolve, job.reject).finally(() => { jobs.delete(job.id); fetchActive = null; pumpFetch(); }); } async function runFetch(job) { const { id } = job; const prefix = `${id}-${now()}`; const base = join(tmpDir, prefix); await job.persisted; try { await setStatus(job, 'downloading'); const info = await getInfo(id); const live = info.is_live || ['is_live', 'post_live', 'is_upcoming'].includes(info.live_status); if (live) throw new MediaSkip('live streams are not cached'); const expected = typeof info.duration === 'number' ? info.duration : 0; const limit = job.auto ? autoMaxSeconds : saveMaxSeconds; if (expected > limit) { throw new MediaSkip(`too long to cache (${Math.round(expected / 60)} min, limit ${Math.round(limit / 60)} min)`); } await makeRoom(id, Math.max(expected, 60) * EST_BYTES_PER_SEC); const raw = await download(id, `${base}.dl.mp4`); await setStatus(job, 'validating'); // Normalise: first video + first audio, moov atom up front so the copy // starts instantly under Range requests. Stream copy — no quality loss. const norm = `${base}.norm.mp4`; const rm = await run(ffmpeg, ['-v', 'error', '-nostdin', '-y', '-i', raw, '-map', '0:v:0', '-map', '0:a:0', '-c', 'copy', '-movflags', '+faststart', '-f', 'mp4', norm]); if (rm.code !== 0) throw new Error('remux failed: ' + tail(rm.stderr)); const probe = await validateMedia(norm, expected, { ffmpeg, ffprobe }); const m4a = `${base}.norm.m4a`; const ex = await run(ffmpeg, ['-v', 'error', '-nostdin', '-y', '-i', norm, '-map', '0:a:0', '-c', 'copy', '-movflags', '+faststart', '-f', 'mp4', m4a]); if (ex.code !== 0 || sizeOf(m4a) < 1024) throw new Error('audio sidecar failed: ' + tail(ex.stderr)); const prev = await db.getMedia(id); const gen = ((prev && prev.gen) || 0) + 1; renameSync(norm, fileFor(id, gen)); renameSync(m4a, fileFor(id, gen, 'm4a')); const t = now(); const fields = { status: 'ready', gen, size: probe.size + sizeOf(fileFor(id, gen, 'm4a')), height: probe.height, vcodec: probe.vcodec, acodec: probe.acodec, duration: probe.duration, optimized: 0, meta: JSON.stringify(metaFromInfo(info, probe.duration)), attempts: 0, error: null, retry_at: 0, updated_at: t, last_access: t, }; await db.upsertMedia(id, fields); removeFiles(id, gen); job.status = 'ready'; log.info?.(`[media] cached ${id} ${probe.height}p ${fmtMB(fields.size)}`); enqueueOptimize(id); return { ...(prev || {}), video_id: id, ...fields }; } catch (err) { job.status = 'failed'; if (err instanceof MediaSkip) { await db.deleteMedia(id).catch(() => {}); log.info?.(`[media] skipped ${id}: ${err.message}`); } else { const prev = await db.getMedia(id).catch(() => null); const attempts = ((prev && prev.attempts) || 0) + 1; await db.upsertMedia(id, { status: 'failed', attempts, error: String(err.message || err).slice(0, 500), retry_at: now() + Math.min(DAY_MS, 15 * 60_000 * 2 ** (attempts - 1)), updated_at: now(), }).catch(() => {}); log.warn?.(`[media] ${id} failed (attempt ${attempts}): ${err.message}`); } throw err; } finally { sweepTmp(prefix); } } // ---- Compression lane ---------------------------------------------------- // CRF (quality-targeted, not a fixed bitrate) at a slow preset is where // small files come from — static lyric/slide videos collapse to a fraction // of YouTube's bitrate. 60 fps is capped to 30. Audio is copied untouched. // Output stays 8-bit H.264: 10-bit / HEVC / AV1 are smaller but do not // play in iOS Safari's