josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sample two short windows before re-encoding a cached video and skip the full encode when HEVC will not save 15 percent

Browse Source
This commit is contained in:
Jonathan Sykes
2026-09-14 00:51:14 +08:00
parent 8b0060d42a
commit e4ac04cb5c
3 changed files with 97 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
server/media-cache.js
View File

@@ -42,6 +42,8 @@ const EVICT_PROTECT_MS = 10 * 60_000; // never evict something played this re
const TOUCH_DEBOUNCE_MS = 60_000;
const EST_BYTES_PER_SEC = 250 * 1024; // ~2 Mbps: pre-compression 720p + 128k AAC
const MIN_SAVING = 0.15; // keep a re-encode only if ≥15% smaller
const SAMPLE_SECONDS = 10; // per sample window (two windows)
const SAMPLE_MIN_DURATION = 60; // shorter videos are cheap enough to encode whole
// x265 tuning from the codec benchmark: stronger adaptive quantisation and
// psy settings, no SAO blur, lighter deblock, longer lookahead.
const X265_TUNING = 'aq-mode=3:aq-strength=0.8:psy-rd=1.5:psy-rdoq=1.0:no-sao=1:deblock=-1,-1:'
@@ -385,14 +387,29 @@ export function createMediaCache({
const hevc = transcode.codec !== 'h264';
const prefix = `${id}-${now()}-${hevc ? 'x265' : 'x264'}`;
const out = join(tmpDir, `${prefix}.mp4`);
const threads = String(transcode.threads);
const video = hevc
? ['-c:v', 'libx265', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
'-pix_fmt', 'yuv420p', '-tag:v', 'hvc1',
'-x265-params', `log-level=error:pools=${threads}:${X265_TUNING}`]
: ['-c:v', 'libx264', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
'-profile:v', 'high', '-level:v', '4.0', '-pix_fmt', 'yuv420p', '-threads', threads];
try {
const threads = String(transcode.threads);
const video = hevc
? ['-c:v', 'libx265', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
'-pix_fmt', 'yuv420p', '-tag:v', 'hvc1',
'-x265-params', `log-level=error:pools=${threads}:${X265_TUNING}`]
: ['-c:v', 'libx264', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
'-profile:v', 'high', '-level:v', '4.0', '-pix_fmt', 'yuv420p', '-threads', threads];
// Sample first. Re-encoding YouTube's already-compressed H.264 usually
// needs MORE bits than YouTube spent (measured on prod: 0 of 5 videos
// shrank at CRF 28; one grew 25%), so spend ~20 s of video finding out
// before ~2x realtime of CPU on the whole thing.
if (row.duration >= SAMPLE_MIN_DURATION) {
const ratio = await sampleRatio(src, row.duration, video, prefix);
const mp4 = sizeOf(src);
const videoBytes = Math.max(0, mp4 - sizeOf(fileFor(id, row.gen, 'm4a')));
const predicted = mp4 - videoBytes * (1 - ratio);
if (predicted > mp4 * (1 - MIN_SAVING)) {
await markDone();
log.info?.(`[media] kept original ${id}: sample predicts ${Math.round(predicted / mp4 * 100)}% of ${fmtMB(mp4)}`);
return;
}
}
const args = ['-v', 'error', '-nostdin', '-y', '-i', src,
'-map', '0:v:0', '-map', '0:a:0',
'-vf', 'scale=-2:min(720\\,ih)',
@@ -429,6 +446,34 @@ export function createMediaCache({
}
}
// Encode two short windows (at 25% and 60% of the video) and compare them
// with the source's own video bytes over exactly the same timestamps.
// Returns encoded/source (below 1 = smaller).
async function sampleRatio(src, duration, videoArgs, prefix) {
let srcBytes = 0;
let encBytes = 0;
const starts = [0.25, 0.6].map((f) => Math.floor(duration * f));
for (const [i, start] of starts.entries()) {
const end = start + SAMPLE_SECONDS;
const pr = await run(ffprobe, ['-v', 'error', '-select_streams', 'v:0',
'-read_intervals', `${Math.max(0, start - 10)}%${end + 1}`,
'-show_entries', 'packet=pts_time,size', '-of', 'csv=p=0', src]);
if (pr.code !== 0) throw new Error('sample probe failed: ' + tail(pr.stderr));
for (const line of pr.stdout.split('\n')) {
const [pts, size] = line.split(',');
const t = parseFloat(pts);
if (t >= start && t < end) srcBytes += parseInt(size, 10) || 0;
}
const out = join(tmpDir, `${prefix}.sample${i}.mp4`);
const args = ['-v', 'error', '-nostdin', '-y', '-ss', String(start), '-t', String(SAMPLE_SECONDS), '-i', src,
'-map', '0:v:0', '-an', '-vf', 'scale=-2:min(720\\,ih)', ...videoArgs, '-fpsmax', '30', '-f', 'mp4', out];
const r = nice ? await run(nice, ['-n', '19', ffmpeg, ...args]) : await run(ffmpeg, args);
if (r.code !== 0) throw new Error('sample encode failed: ' + tail(r.stderr));
encBytes += sizeOf(out);
}
return srcBytes > 0 ? encBytes / srcBytes : 1;
}
// ---- Public API ----------------------------------------------------------
// Resolve to the ready row, fetching it first if needed. Rejects with a

35
server/media-cache.test.js
View File

@@ -38,6 +38,13 @@ beforeAll(async () => {
'-c:v', 'libx265', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p', '-tag:v', 'hvc1', '-x265-params', 'log-level=error',
'-c:a', 'aac', '-shortest', fx('hevc.mp4'));
ff('-i', fx('hevc.mp4'), '-c', 'copy', '-tag:v', 'hev1', fx('hevc-hev1.mp4'));
// 64 s sources for the sample-first path (videos ≥ 60 s get sampled):
// "fat" = far more bits than needed (HEVC will win), "lean" = already
// starved (HEVC at CRF 28 can't beat it — like most YouTube encodes).
ff('-f', 'lavfi', '-i', 'testsrc2=size=640x360:rate=24:duration=64', '-f', 'lavfi', '-i', 'sine=frequency=330:duration=64',
'-c:v', 'libx264', '-preset', 'ultrafast', '-qp', '8', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-shortest', fx('fat64.mp4'));
ff('-f', 'lavfi', '-i', 'testsrc2=size=640x360:rate=24:duration=64', '-f', 'lavfi', '-i', 'sine=frequency=330:duration=64',
'-c:v', 'libx264', '-preset', 'medium', '-crf', '44', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-shortest', fx('lean64.mp4'));
const buf = readFileSync(fx('good.mp4'));
writeFileSync(fx('truncated.mp4'), buf.subarray(0, Math.floor(buf.length * 0.6)));
// The nasty case: moov up front (index says 10 s) but the media data is cut.
@@ -241,6 +248,34 @@ describe('media cache jobs', () => {
expect(await cache.verify('optAAAAAAA1')).toBe(true);
});
test('sample-first: a source HEVC cannot beat is skipped without a full encode', async () => {
await clearDb();
const lines = [];
const { cache } = makeCache({ fixture: 'lean64.mp4', duration: 64, log: { info: (m) => lines.push(m), warn: (m) => lines.push(m) },
transcode: { enabled: true, codec: 'hevc', crf: 28, preset: 'ultrafast', threads: 2 } });
await cache.init();
const first = await cache.ensureCached('sampAAAAAA1', { priority: HIGH });
expect(await until(async () => (await dbmod.getMedia('sampAAAAAA1')).optimized === OPT_REV, 60000)).toBe(true);
const after = await dbmod.getMedia('sampAAAAAA1');
expect(after.gen).toBe(first.gen);
expect(after.vcodec).toBe('h264');
expect(lines.some((l) => /sample predicts/.test(l))).toBe(true);
}, 90000);
test('sample-first: a source HEVC can beat gets the full encode and swaps', async () => {
await clearDb();
const lines = [];
const { cache } = makeCache({ fixture: 'fat64.mp4', duration: 64, log: { info: (m) => lines.push(m), warn: (m) => lines.push(m) },
transcode: { enabled: true, codec: 'hevc', crf: 28, preset: 'ultrafast', threads: 2 } });
await cache.init();
const first = await cache.ensureCached('sampAAAAAA2', { priority: HIGH });
expect(await until(async () => (await dbmod.getMedia('sampAAAAAA2')).optimized === OPT_REV, 60000)).toBe(true);
const after = await dbmod.getMedia('sampAAAAAA2');
expect(after.vcodec).toBe('hevc');
expect(after.gen).toBeGreaterThan(first.gen);
expect(lines.some((l) => /sample predicts/.test(l))).toBe(false);
}, 90000);
test('copies evaluated by an older lane revision are re-queued at boot', async () => {
await clearDb();
const seed = makeCache();