// bun test media-cache.test.js — validation gate, job queue, eviction, redownload. // Uses real ffmpeg fixtures and the real db.js against a throwaway DB file. import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { spawnSync } from 'node:child_process'; import { mkdtempSync, rmSync, copyFileSync, writeFileSync, readFileSync, existsSync, readdirSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; const root = mkdtempSync(join(tmpdir(), 'ytp-media-test-')); process.env.DB_PATH = join(root, 'test.db'); const dbmod = await import('./db.js'); const { createMediaCache, validateMedia, MediaSkip, HIGH, LOW, OPT_REV } = await import('./media-cache.js'); const { initP2pSchema } = await import('./p2p-db.js'); const { sha256File } = await import('./hash.js'); const fx = (name) => join(root, name); const ff = (...args) => { const r = spawnSync('ffmpeg', ['-v', 'error', '-y', ...args]); if (r.status !== 0) throw new Error(String(r.stderr)); }; const silentLog = { info() {}, warn() {} }; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); async function until(fn, ms = 20000) { const end = Date.now() + ms; while (Date.now() < end) { if (await fn()) return true; await sleep(50); } return false; } beforeAll(async () => { await dbmod.initDb(); await initP2pSchema(); // adds media_cache.sha256 // 10 s 640x360 H.264 + AAC — the shape of a good cached copy. ff('-f', 'lavfi', '-i', 'testsrc=size=640x360:rate=30:duration=10', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=10', '-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-shortest', fx('good.mp4')); ff('-f', 'lavfi', '-i', 'testsrc=size=640x360:rate=30:duration=10', '-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p', fx('noaudio.mp4')); ff('-f', 'lavfi', '-i', 'testsrc=size=640x360:rate=30:duration=10', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=10', '-c:v', 'mpeg4', '-c:a', 'aac', '-shortest', fx('mpeg4.mp4')); // HEVC as the compression lane writes it (hvc1) and as Safari rejects it (hev1). ff('-f', 'lavfi', '-i', 'testsrc=size=640x360:rate=30:duration=10', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=10', '-c:v', 'libx265', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p', '-tag:v', 'hvc1', '-x265-params', 'log-level=error', '-c:a', 'aac', '-shortest', fx('hevc.mp4')); ff('-i', fx('hevc.mp4'), '-c', 'copy', '-tag:v', 'hev1', fx('hevc-hev1.mp4')); // 64 s sources for the sample-first path (videos ≥ 60 s get sampled): // "fat" = far more bits than needed (HEVC will win), "lean" = already // starved (HEVC at CRF 28 can't beat it — like most YouTube encodes). ff('-f', 'lavfi', '-i', 'testsrc2=size=640x360:rate=24:duration=64', '-f', 'lavfi', '-i', 'sine=frequency=330:duration=64', '-c:v', 'libx264', '-preset', 'ultrafast', '-qp', '8', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-shortest', fx('fat64.mp4')); ff('-f', 'lavfi', '-i', 'testsrc2=size=640x360:rate=24:duration=64', '-f', 'lavfi', '-i', 'sine=frequency=330:duration=64', '-c:v', 'libx264', '-preset', 'medium', '-crf', '44', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-shortest', fx('lean64.mp4')); const buf = readFileSync(fx('good.mp4')); writeFileSync(fx('truncated.mp4'), buf.subarray(0, Math.floor(buf.length * 0.6))); // The nasty case: moov up front (index says 10 s) but the media data is cut. ff('-i', fx('good.mp4'), '-c', 'copy', '-movflags', '+faststart', fx('good-fs.mp4')); const fs = readFileSync(fx('good-fs.mp4')); writeFileSync(fx('truncated-fs.mp4'), fs.subarray(0, Math.floor(fs.length * 0.6))); }); afterAll(() => rmSync(root, { recursive: true, force: true })); describe('validateMedia', () => { test('accepts a good H.264 + AAC file', async () => { const p = await validateMedia(fx('good.mp4'), 10); expect(p.vcodec).toBe('h264'); expect(p.acodec).toBe('aac'); expect(p.height).toBe(360); }); test('rejects a truncated file', async () => { await expect(validateMedia(fx('truncated.mp4'), 10)).rejects.toThrow(/validation/); }); test('rejects a faststart file whose media data is cut short', async () => { await expect(validateMedia(fx('truncated-fs.mp4'), 10)).rejects.toThrow(/validation/); }); test('rejects a file without audio', async () => { await expect(validateMedia(fx('noaudio.mp4'), 10)).rejects.toThrow(/AAC/); }); test('rejects a non-h264 file', async () => { await expect(validateMedia(fx('mpeg4.mp4'), 10)).rejects.toThrow(/not h264/); }); test('accepts hvc1-tagged HEVC for the compression lane, but not as a download', async () => { const p = await validateMedia(fx('hevc.mp4'), 10); expect(p.vcodec).toBe('hevc'); await expect(validateMedia(fx('hevc.mp4'), 10, { codecs: ['h264'] })).rejects.toThrow(/not h264/); }); test('rejects hev1-tagged HEVC (Safari refuses it)', async () => { await expect(validateMedia(fx('hevc-hev1.mp4'), 10)).rejects.toThrow(/hvc1/); }); test('rejects a file shorter than the source video', async () => { await expect(validateMedia(fx('good.mp4'), 60)).rejects.toThrow(/truncated/); }); }); let dirN = 0; function makeCache({ fixture = 'good.mp4', duration = 10, ...opts } = {}) { const dir = join(root, 'media' + dirN++); const calls = []; const db = { getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia, listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia, mediaStats: dbmod.mediaStats, }; const state = { fixture }; const cache = createMediaCache({ dir, db, log: silentLog, getInfo: async (id) => ({ id, title: 'T ' + id, channel: 'C', duration }), download: async (id, out) => { calls.push(id); await sleep(30); copyFileSync(fx(state.fixture), out); return out; }, transcode: { enabled: false }, freeBytes: () => 100 * 1024 ** 3, backfillDelayMs: -1, ...opts, }); return { cache, dir, calls, state }; } async function clearDb() { for (const r of await dbmod.listMedia()) await dbmod.deleteMedia(r.video_id); } describe('content hashes (P2P cids)', () => { test('a promoted copy stores the sha256 of its mp4 and reports it once', async () => { await clearDb(); const seen = []; const { cache, dir } = makeCache({ onReady: (i) => { seen.push(i); } }); await cache.init(); const row = await cache.ensureCached('hashAAAAAA1', { priority: HIGH }); const want = await sha256File(join(dir, `hashAAAAAA1.${row.gen}.mp4`)); expect((await dbmod.getMedia('hashAAAAAA1')).sha256).toBe(want); expect(await until(() => seen.length === 1)).toBe(true); expect(seen[0]).toMatchObject({ id: 'hashAAAAAA1', gen: row.gen, sha256: want, vcodec: 'h264' }); expect(seen[0].meta.title).toBe('T hashAAAAAA1'); }); test('backfill hashes ready copies that have no sha256 yet', async () => { await clearDb(); const seen = []; const { cache } = makeCache({ onReady: (i) => { seen.push(i); } }); await cache.init(); await cache.ensureCached('hashAAAAAA2', { priority: HIGH }); await dbmod.upsertMedia('hashAAAAAA2', { sha256: null }); seen.length = 0; expect(await cache.backfillHashes()).toBe(1); expect((await dbmod.getMedia('hashAAAAAA2')).sha256).toMatch(/^[0-9a-f]{64}$/); expect(await until(() => seen.length === 1)).toBe(true); expect(await cache.backfillHashes()).toBe(0); // nothing left }); }); describe('media cache jobs', () => { test('fetches once, dedupes concurrent requests, then serves from disk', async () => { await clearDb(); const { cache, dir, calls } = makeCache(); await cache.init(); const [a, b] = await Promise.all([ cache.ensureCached('vidAAAAAAA1', { priority: LOW, auto: true }), cache.ensureCached('vidAAAAAAA1', { priority: HIGH }), ]); expect(calls).toEqual(['vidAAAAAAA1']); expect(a.status).toBe('ready'); expect(b.gen).toBe(a.gen); expect(existsSync(join(dir, `vidAAAAAAA1.${a.gen}.mp4`))).toBe(true); expect(existsSync(join(dir, `vidAAAAAAA1.${a.gen}.m4a`))).toBe(true); expect(await cache.getReady('vidAAAAAAA1')).not.toBeNull(); await cache.ensureCached('vidAAAAAAA1'); expect(calls.length).toBe(1); expect(JSON.parse((await cache.getReady('vidAAAAAAA1')).meta).title).toBe('T vidAAAAAAA1'); }); test('HIGH request jumps ahead of queued LOW jobs', async () => { await clearDb(); const { cache, calls } = makeCache(); await cache.init(); const p = [ cache.ensureCached('lowAAAAAAA1', { auto: true }), cache.ensureCached('lowAAAAAAA2', { auto: true }), cache.ensureCached('lowAAAAAAA3', { auto: true }), cache.ensureCached('highAAAAAA1', { priority: HIGH }), ]; await Promise.all(p); expect(calls[0]).toBe('lowAAAAAAA1'); // already running when HIGH arrived expect(calls[1]).toBe('highAAAAAA1'); }); test('a broken download is never cached and backs off', async () => { await clearDb(); const { cache, dir, calls } = makeCache({ fixture: 'truncated.mp4' }); await cache.init(); await expect(cache.ensureCached('brokenAAAA1', { priority: HIGH })).rejects.toThrow(/validation|remux/); const row = await dbmod.getMedia('brokenAAAA1'); expect(row.status).toBe('failed'); expect(row.retry_at).toBeGreaterThan(Date.now()); expect(readdirSync(dir).filter((n) => n !== '.tmp')).toEqual([]); expect(readdirSync(join(dir, '.tmp'))).toEqual([]); await expect(cache.ensureCached('brokenAAAA1', { priority: HIGH })).rejects.toMatchObject({ code: 'BACKOFF' }); expect(calls.length).toBe(1); }); test('a faststart download with cut media data is rejected too', async () => { await clearDb(); const { cache, dir } = makeCache({ fixture: 'truncated-fs.mp4' }); await cache.init(); await expect(cache.ensureCached('brokenAAAA2', { priority: HIGH })).rejects.toThrow(); expect((await dbmod.getMedia('brokenAAAA2')).status).toBe('failed'); expect(readdirSync(dir).filter((n) => n !== '.tmp')).toEqual([]); }); test('redownload drops the copy and fetches a fresh one', async () => { await clearDb(); const { cache, dir, calls } = makeCache(); await cache.init(); const first = await cache.ensureCached('redoAAAAAA1', { priority: HIGH }); const res = await cache.redownload('redoAAAAAA1'); expect(res.status).toBe('queued'); expect(existsSync(join(dir, `redoAAAAAA1.${first.gen}.mp4`))).toBe(false); expect(await cache.getReady('redoAAAAAA1')).toBeNull(); expect(await until(async () => !!(await cache.getReady('redoAAAAAA1')))).toBe(true); const second = await cache.getReady('redoAAAAAA1'); expect(second.gen).toBeGreaterThan(first.gen); expect(calls.length).toBe(2); }); test('auto jobs respect the auto length limit; explicit saves do not', async () => { await clearDb(); const { cache } = makeCache({ duration: 7200, autoMaxSeconds: 3600 }); await cache.init(); await expect(cache.ensureCached('longAAAAAA1', { auto: true })).rejects.toMatchObject({ code: 'SKIPPED' }); expect(await dbmod.getMedia('longAAAAAA1')).toBeNull(); }); test('evicts least-recently-played copies to stay under the budget', async () => { await clearDb(); const size = readFileSync(fx('good.mp4')).length; // Budget fits ~2 copies (+ sidecars) but not 3 once the new-download estimate is added. const { cache } = makeCache({ maxBytes: size * 2 + 60 * 250 * 1024 + 50_000, duration: 10 }); await cache.init(); await cache.ensureCached('lruAAAAAAA1', { priority: HIGH }); await cache.ensureCached('lruAAAAAAA2', { priority: HIGH }); // Age both beyond the eviction protection window; #1 is the older one. await dbmod.upsertMedia('lruAAAAAAA1', { last_access: 1000 }); await dbmod.upsertMedia('lruAAAAAAA2', { last_access: 2000 }); await cache.ensureCached('lruAAAAAAA3', { priority: HIGH }); expect(await dbmod.getMedia('lruAAAAAAA1')).toBeNull(); expect(await cache.getReady('lruAAAAAAA3')).not.toBeNull(); }); test('free-disk guard skips caching', async () => { await clearDb(); const { cache, calls } = makeCache({ freeBytes: () => 1024 ** 3, minFreeBytes: 5 * 1024 ** 3 }); await cache.init(); await expect(cache.ensureCached('diskAAAAAA1', { priority: HIGH })).rejects.toMatchObject({ code: 'SKIPPED' }); expect(calls.length).toBe(0); }); test('boot resumes a job left queued and sweeps stray files', async () => { await clearDb(); const { cache, dir, calls } = makeCache(); await dbmod.upsertMedia('bootAAAAAA1', { status: 'downloading', priority: HIGH }); writeFileSync(join(dir + '_never'), ''); const { mkdirSync } = await import('node:fs'); mkdirSync(join(dir, '.tmp'), { recursive: true }); writeFileSync(join(dir, '.tmp', 'partial.mp4.part'), 'x'); writeFileSync(join(dir, 'orphanAAAA1.3.mp4'), 'x'); await cache.init(); expect(existsSync(join(dir, 'orphanAAAA1.3.mp4'))).toBe(false); expect(await until(async () => !!(await cache.getReady('bootAAAAAA1')))).toBe(true); expect(calls).toEqual(['bootAAAAAA1']); }); test('HEVC compression lane swaps in a smaller copy under a new gen', async () => { await clearDb(); const { cache } = makeCache({ transcode: { enabled: true, codec: 'hevc', crf: 34, preset: 'ultrafast', threads: 2 } }); await cache.init(); const first = await cache.ensureCached('optAAAAAAA1', { priority: HIGH }); expect(first.vcodec).toBe('h264'); expect(await until(async () => (await dbmod.getMedia('optAAAAAAA1')).optimized === OPT_REV)).toBe(true); const after = await dbmod.getMedia('optAAAAAAA1'); expect(after.gen).toBeGreaterThan(first.gen); expect(after.vcodec).toBe('hevc'); expect(after.size).toBeLessThan(first.size); // old generation is kept for in-flight playback (grace period) expect(await cache.filePath('optAAAAAAA1', first.gen)).not.toBeNull(); expect(await cache.filePath('optAAAAAAA1', after.gen, 'm4a')).not.toBeNull(); // and the HEVC copy still passes the gate verify() uses expect(await cache.verify('optAAAAAAA1')).toBe(true); }); test('sample-first: a source HEVC cannot beat is skipped without a full encode', async () => { await clearDb(); const lines = []; const { cache } = makeCache({ fixture: 'lean64.mp4', duration: 64, log: { info: (m) => lines.push(m), warn: (m) => lines.push(m) }, transcode: { enabled: true, codec: 'hevc', crf: 28, preset: 'ultrafast', threads: 2 } }); await cache.init(); const first = await cache.ensureCached('sampAAAAAA1', { priority: HIGH }); expect(await until(async () => (await dbmod.getMedia('sampAAAAAA1')).optimized === OPT_REV, 60000)).toBe(true); const after = await dbmod.getMedia('sampAAAAAA1'); expect(after.gen).toBe(first.gen); expect(after.vcodec).toBe('h264'); expect(lines.some((l) => /sample predicts/.test(l))).toBe(true); }, 90000); test('sample-first: a source HEVC can beat gets the full encode and swaps', async () => { await clearDb(); const lines = []; const { cache } = makeCache({ fixture: 'fat64.mp4', duration: 64, log: { info: (m) => lines.push(m), warn: (m) => lines.push(m) }, transcode: { enabled: true, codec: 'hevc', crf: 28, preset: 'ultrafast', threads: 2 } }); await cache.init(); const first = await cache.ensureCached('sampAAAAAA2', { priority: HIGH }); expect(await until(async () => (await dbmod.getMedia('sampAAAAAA2')).optimized === OPT_REV, 60000)).toBe(true); const after = await dbmod.getMedia('sampAAAAAA2'); expect(after.vcodec).toBe('hevc'); expect(after.gen).toBeGreaterThan(first.gen); expect(lines.some((l) => /sample predicts/.test(l))).toBe(false); }, 90000); test('copies evaluated by an older lane revision are re-queued at boot', async () => { await clearDb(); const seed = makeCache(); await seed.cache.init(); await seed.cache.ensureCached('revAAAAAAA1', { priority: HIGH }); await dbmod.upsertMedia('revAAAAAAA1', { optimized: 1 }); // the old x264 revision // A fresh instance over the same dir + DB, as after a deploy. const same = createMediaCache({ dir: seed.dir, db: { getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia, listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia, mediaStats: dbmod.mediaStats }, log: silentLog, getInfo: async () => ({ duration: 10 }), download: async () => { throw new Error('no download expected'); }, transcode: { enabled: true, codec: 'hevc', crf: 34, preset: 'ultrafast', threads: 2 }, freeBytes: () => 100 * 1024 ** 3, }); await same.init(); expect(await until(async () => (await dbmod.getMedia('revAAAAAAA1')).optimized === OPT_REV)).toBe(true); }); test('verify() drops and refetches a copy that went bad on disk, keeps a good one', async () => { await clearDb(); let clock = Date.now(); const { cache, dir, calls } = makeCache({ now: () => clock }); await cache.init(); const row = await cache.ensureCached('verAAAAAAA1', { priority: HIGH }); expect(await cache.verify('verAAAAAAA1')).toBe(true); writeFileSync(join(dir, `verAAAAAAA1.${row.gen}.mp4`), Buffer.alloc(300_000, 7)); expect(await cache.verify('verAAAAAAA1')).toBeNull(); // debounced for 10 min clock += 11 * 60_000; expect(await cache.verify('verAAAAAAA1')).toBe(false); expect(await until(async () => !!(await cache.getReady('verAAAAAAA1')))).toBe(true); expect((await cache.getReady('verAAAAAAA1')).gen).toBeGreaterThan(row.gen); expect(calls.length).toBe(2); }); test('rejects path-like ids', async () => { const { cache } = makeCache(); await expect(cache.ensureCached('../../etc')).rejects.toBeInstanceOf(MediaSkip); expect(await cache.filePath('..', 1)).toBeNull(); }); });