josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re-encode cached videos to HEVC when it saves at least 15 percent and serve them only to devices that can decode HEVC

Browse Source
This commit is contained in:
Jonathan Sykes
2026-09-13 21:09:15 +08:00
parent 91dab289c3
commit 8b0060d42a
7 changed files with 159 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
server/media-cache.js
View File

@@ -28,6 +28,11 @@ import { join } from 'node:path';
export const HIGH = 0; // explicit save / Broken re-download
export const LOW = 1; // auto-cache after a play
// Compression-lane revision. `optimized` stores the revision a copy was last
// evaluated with; bumping this re-queues every cached copy once (rev 1 was
// x264, which never beat YouTube's own encode; rev 2 is HEVC).
export const OPT_REV = 2;
const ID_RE = /^[A-Za-z0-9_-]{6,64}$/;
export const isMediaId = (id) => typeof id === 'string' && ID_RE.test(id);
@@ -36,7 +41,11 @@ const OLD_GEN_GRACE_MS = 30 * 60_000; // keep superseded files for in-flight
const EVICT_PROTECT_MS = 10 * 60_000; // never evict something played this recently
const TOUCH_DEBOUNCE_MS = 60_000;
const EST_BYTES_PER_SEC = 250 * 1024; // ~2 Mbps: pre-compression 720p + 128k AAC
const MIN_SAVING = 0.15; // keep an x264 re-encode only if ≥15% smaller
const MIN_SAVING = 0.15; // keep a re-encode only if ≥15% smaller
// x265 tuning from the codec benchmark: stronger adaptive quantisation and
// psy settings, no SAO blur, lighter deblock, longer lookahead.
const X265_TUNING = 'aq-mode=3:aq-strength=0.8:psy-rd=1.5:psy-rdoq=1.0:no-sao=1:deblock=-1,-1:'
+ 'bframes=8:rc-lookahead=60:ref=6:no-strong-intra-smoothing=1';
const DAY_MS = 24 * 3600_000;
// A condition that means "don't cache this right now, just stream it" — not a
@@ -69,20 +78,22 @@ function run(cmd, args) {
// ----------------------------------------------------------------------------
// Validation gate. Throws with a reason; resolves to probe facts on success.
// 1. file exists and is not tiny
// 2. exactly one H.264 8-bit 4:2:0 video stream + an AAC audio stream
// (AV1/VP9/10-bit play on almost nothing Apple — see SAVE_MUX_FORMAT)
// 2. exactly one 8-bit 4:2:0 video stream in an allowed codec + an AAC
// audio stream. Downloads must be H.264 (AV1/VP9 play on almost nothing
// Apple — see SAVE_MUX_FORMAT); the compression lane may produce HEVC,
// which must carry the `hvc1` tag or Safari refuses it.
// 3. container AND per-stream durations cover the expected length
// 4. full demux pass reports no errors (catches corrupt/truncated boxes)
// 5. the first and last seconds actually decode
// ----------------------------------------------------------------------------
export async function validateMedia(path, expected, { ffmpeg = 'ffmpeg', ffprobe = 'ffprobe' } = {}) {
export async function validateMedia(path, expected, { ffmpeg = 'ffmpeg', ffprobe = 'ffprobe', codecs = ['h264', 'hevc'] } = {}) {
const size = sizeOf(path);
if (!size) throw new Error('validation: output file missing');
if (size < MIN_BYTES) throw new Error(`validation: file too small (${size} bytes)`);
const pr = await run(ffprobe, [
'-v', 'error',
'-show_entries', 'format=duration:stream=codec_type,codec_name,height,pix_fmt,duration:stream_disposition=attached_pic',
'-show_entries', 'format=duration:stream=codec_type,codec_name,codec_tag_string,height,pix_fmt,duration:stream_disposition=attached_pic',
'-of', 'json', path,
]);
if (pr.code !== 0) throw new Error('validation: ffprobe failed: ' + tail(pr.stderr));
@@ -93,7 +104,10 @@ export async function validateMedia(path, expected, { ffmpeg = 'ffmpeg', ffprobe
const audio = streams.filter((s) => s.codec_type === 'audio');
if (video.length !== 1) throw new Error(`validation: expected 1 video stream, found ${video.length}`);
const v = video[0];
if (v.codec_name !== 'h264') throw new Error(`validation: video codec ${v.codec_name} is not h264`);
if (!codecs.includes(v.codec_name)) throw new Error(`validation: video codec ${v.codec_name} is not ${codecs.join('/')}`);
if (v.codec_name === 'hevc' && v.codec_tag_string !== 'hvc1') {
throw new Error(`validation: HEVC tagged ${v.codec_tag_string} (Safari needs hvc1)`);
}
if (v.pix_fmt && !/^yuvj?420p$/.test(v.pix_fmt)) throw new Error(`validation: pixel format ${v.pix_fmt} is not 8-bit 4:2:0`);
const a = audio.find((s) => s.codec_name === 'aac');
if (!a) throw new Error('validation: no AAC audio stream');
@@ -147,7 +161,7 @@ export function createMediaCache({
minFreeBytes = 5 * 1024 ** 3,
autoMaxSeconds = 3600,
saveMaxSeconds = 3 * 3600,
transcode = { enabled: true, crf: 28, preset: 'slow', threads: 2 },
transcode = { enabled: true, codec: 'hevc', crf: 28, preset: 'medium', threads: 2, maxSeconds: 3600 },
freeBytes = () => { const s = statfsSync(dir); return s.bavail * s.bsize; },
now = () => Date.now(),
log = console,
@@ -285,7 +299,7 @@ export function createMediaCache({
const rm = await run(ffmpeg, ['-v', 'error', '-nostdin', '-y', '-i', raw,
'-map', '0:v:0', '-map', '0:a:0', '-c', 'copy', '-movflags', '+faststart', '-f', 'mp4', norm]);
if (rm.code !== 0) throw new Error('remux failed: ' + tail(rm.stderr));
const probe = await validateMedia(norm, expected, { ffmpeg, ffprobe });
const probe = await validateMedia(norm, expected, { ffmpeg, ffprobe, codecs: ['h264'] });
const m4a = `${base}.norm.m4a`;
const ex = await run(ffmpeg, ['-v', 'error', '-nostdin', '-y', '-i', norm,
@@ -332,11 +346,15 @@ export function createMediaCache({
}
// ---- Compression lane ----------------------------------------------------
// CRF (quality-targeted, not a fixed bitrate) at a slow preset is where
// small files come from — static lyric/slide videos collapse to a fraction
// of YouTube's bitrate. 60 fps is capped to 30. Audio is copied untouched.
// Output stays 8-bit H.264: 10-bit / HEVC / AV1 are smaller but do not
// play in iOS Safari's <video>.
// Re-encodes a ready copy with CRF (quality-targeted) and keeps the result
// only when it is ≥15% smaller. Measured 2026-09-13: x264 can't beat
// YouTube's own H.264 (it needed MORE bitrate for the same VMAF), while
// x265 8-bit medium CRF 28 cut live worship video ~44% (VMAF ~93) at ~2x
// realtime on the homelab's 2 niced threads. HEVC plays on iPhone Safari,
// Android and Chrome/Edge with hardware decode; the server only serves an
// HEVC copy to clients that ask for it (?hevc=1), everyone else streams.
// 8-bit on purpose — the widest hardware-decode support. 60 fps is capped
// to 30; audio is copied untouched.
function enqueueOptimize(id) {
if (!transcode || !transcode.enabled || optPending.has(id)) return;
optPending.add(id);
@@ -355,33 +373,41 @@ export function createMediaCache({
async function runOptimize(id) {
const row = await db.getMedia(id);
if (!row || row.status !== 'ready' || row.optimized) return;
if (!row || row.status !== 'ready' || (row.optimized || 0) >= OPT_REV) return;
const src = fileFor(id, row.gen);
if (!existsSync(src)) return;
const prefix = `${id}-${now()}-x264`;
const out = join(tmpDir, `${prefix}.mp4`);
const markDone = async () => {
const cur = await db.getMedia(id);
if (cur && cur.status === 'ready' && cur.gen === row.gen) await db.upsertMedia(id, { optimized: 1 });
if (cur && cur.status === 'ready' && cur.gen === row.gen) await db.upsertMedia(id, { optimized: OPT_REV });
};
// Very long videos would tie the lane up for hours on 2 niced threads.
if (transcode.maxSeconds && row.duration > transcode.maxSeconds) { await markDone(); return; }
const hevc = transcode.codec !== 'h264';
const prefix = `${id}-${now()}-${hevc ? 'x265' : 'x264'}`;
const out = join(tmpDir, `${prefix}.mp4`);
try {
const threads = String(transcode.threads);
const video = hevc
? ['-c:v', 'libx265', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
'-pix_fmt', 'yuv420p', '-tag:v', 'hvc1',
'-x265-params', `log-level=error:pools=${threads}:${X265_TUNING}`]
: ['-c:v', 'libx264', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
'-profile:v', 'high', '-level:v', '4.0', '-pix_fmt', 'yuv420p', '-threads', threads];
const args = ['-v', 'error', '-nostdin', '-y', '-i', src,
'-map', '0:v:0', '-map', '0:a:0',
'-vf', 'scale=-2:min(720\\,ih)',
'-c:v', 'libx264', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
'-profile:v', 'high', '-level:v', '4.0', '-pix_fmt', 'yuv420p',
'-threads', String(transcode.threads), '-fpsmax', '30',
...video, '-fpsmax', '30',
'-c:a', 'copy', '-movflags', '+faststart', '-f', 'mp4', out];
const r = nice ? await run(nice, ['-n', '19', ffmpeg, ...args]) : await run(ffmpeg, args);
if (r.code !== 0) throw new Error('x264 encode failed: ' + tail(r.stderr));
const probe = await validateMedia(out, row.duration, { ffmpeg, ffprobe });
if (r.code !== 0) throw new Error(`${hevc ? 'x265' : 'x264'} encode failed: ` + tail(r.stderr));
const probe = await validateMedia(out, row.duration, { ffmpeg, ffprobe, codecs: [hevc ? 'hevc' : 'h264'] });
const cur = await db.getMedia(id);
if (!cur || cur.status !== 'ready' || cur.gen !== row.gen) return; // replaced or evicted meanwhile
const oldSize = sizeOf(src);
if (probe.size > oldSize * (1 - MIN_SAVING)) {
await db.upsertMedia(id, { optimized: 1 });
log.info?.(`[media] kept original ${id}: x264 ${fmtMB(probe.size)} vs ${fmtMB(oldSize)}`);
await db.upsertMedia(id, { optimized: OPT_REV });
log.info?.(`[media] kept original ${id}: ${probe.vcodec} ${fmtMB(probe.size)} vs ${fmtMB(oldSize)}`);
return;
}
const gen = row.gen + 1;
@@ -390,13 +416,13 @@ export function createMediaCache({
const newAudio = fileFor(id, gen, 'm4a');
try { linkSync(oldAudio, newAudio); } catch { copyFileSync(oldAudio, newAudio); }
await db.upsertMedia(id, {
gen, size: probe.size + sizeOf(newAudio), height: probe.height,
duration: probe.duration, optimized: 1, updated_at: now(),
gen, size: probe.size + sizeOf(newAudio), height: probe.height, vcodec: probe.vcodec,
duration: probe.duration, optimized: OPT_REV, updated_at: now(),
});
removeFiles(id, gen, OLD_GEN_GRACE_MS);
log.info?.(`[media] optimized ${id}: ${fmtMB(oldSize)} → ${fmtMB(probe.size)}`);
log.info?.(`[media] optimized ${id}: ${fmtMB(oldSize)} → ${fmtMB(probe.size)} (${probe.vcodec})`);
} catch (err) {
await markDone().catch(() => {}); // don't retry a source x264 can't improve
await markDone().catch(() => {}); // don't retry a source the encoder can't improve
throw err;
} finally {
sweepTmp(prefix);
@@ -505,7 +531,8 @@ export function createMediaCache({
status: job ? job.status : row ? row.status : 'none',
size: row ? row.size : 0,
height: row ? row.height : 0,
optimized: !!(row && row.optimized),
vcodec: row ? row.vcodec : null,
optimized: !!(row && (row.optimized || 0) >= OPT_REV),
optimizing: optActive === id,
error: row && row.status === 'failed' ? row.error : null,
retryAt: row && row.status === 'failed' ? row.retry_at : 0,
@@ -542,7 +569,7 @@ export function createMediaCache({
for (const r of rows) {
if (r.status === 'ready') {
if (!readyFilesExist(r)) { await db.deleteMedia(r.video_id); removeFiles(r.video_id); continue; }
if (!r.optimized) enqueueOptimize(r.video_id);
if ((r.optimized || 0) < OPT_REV) enqueueOptimize(r.video_id);
} else if (r.status === 'queued' || r.status === 'downloading' || r.status === 'validating') {
startJob(r.video_id, r.priority === HIGH ? HIGH : LOW, !!r.auto, r);
resumed++;