Re-encode cached videos to HEVC when it saves at least 15 percent and serve them only to devices that can decode HEVC
This commit is contained in:
17
CLAUDE.md
17
CLAUDE.md
@@ -42,10 +42,19 @@ JSON shapes mirror the Tauri Rust bridge exactly — don't change one side alone
|
|||||||
(15 min · 2^n, max 24 h) and saves fall back to the legacy per-request path.
|
(15 min · 2^n, max 24 h) and saves fall back to the legacy per-request path.
|
||||||
- `gen` is in the filename and URL (`/api/media/<id>?g=<gen>`): a URL never serves
|
- `gen` is in the filename and URL (`/api/media/<id>?g=<gen>`): a URL never serves
|
||||||
different bytes. A replaced gen is kept 30 min so in-flight Range playback survives.
|
different bytes. A replaced gen is kept 30 min so in-flight Range playback survives.
|
||||||
- **Compression lane**: niced x264 `-preset slow -crf 28 -fpsmax 30`, audio copied,
|
- **Compression lane** (rev `OPT_REV` = 2): niced x265 8-bit `-preset medium -crf 28`,
|
||||||
kept only if ≥15% smaller. Measured: YouTube's 720p avc1 is already ~0.6–0.7 Mbps, so
|
`hvc1` tag, anime-style tuning (`X265_TUNING`), `-fpsmax 30`, audio copied — kept only
|
||||||
CRF 24 made files *bigger*; CRF 28 ≈ −20% video at SSIM 0.994. Lyric videos usually
|
if ≥15% smaller, and the copy's `vcodec` becomes `hevc`. Benchmarked 2026-09-13 (VMAF,
|
||||||
keep the original. HEVC/AV1/10-bit are smaller but don't play in iOS `<video>`.
|
codec-test page in Windows Docs): x264 can NOT beat YouTube's own H.264 (needs more
|
||||||
|
bitrate at equal quality); x265 cuts live worship video ~35–44% at ~2× realtime on the
|
||||||
|
homelab's 2 threads; lyric videos (~47 kbps video, mostly audio) keep the original.
|
||||||
|
Bumping `OPT_REV` re-queues every cached copy once at boot.
|
||||||
|
- **HEVC gating**: an HEVC copy is served only to clients that send `?hevc=1` on
|
||||||
|
`/api/streams` and `/api/download` (app.js `hevcCapable()` = canPlayType hvc1, minus a
|
||||||
|
per-device `ytpNoHevc` flag set when an HEVC copy fails to play). Everyone else gets
|
||||||
|
the pre-cache behaviour (YouTube proxy / legacy H.264 save). Verified: iPhone XR, Galaxy
|
||||||
|
S10, Windows Chrome play HEVC; AV1 fails on the XR (no decoder), so it isn't used.
|
||||||
|
The validation gate requires `hvc1` (Safari refuses `hev1`) and 8-bit 4:2:0.
|
||||||
- **Budget**: `MEDIA_CACHE_MAX_BYTES` (10 GiB) LRU by `last_access` (anything played in
|
- **Budget**: `MEDIA_CACHE_MAX_BYTES` (10 GiB) LRU by `last_access` (anything played in
|
||||||
the last 10 min is protected) + `MEDIA_MIN_FREE_BYTES` (5 GiB) disk guard → skip, stream.
|
the last 10 min is protected) + `MEDIA_MIN_FREE_BYTES` (5 GiB) disk guard → skip, stream.
|
||||||
- **Broken**: `POST /api/media/:id/redownload` (now-playing ⚠ Broken button). When a
|
- **Broken**: `POST /api/media/:id/redownload` (now-playing ⚠ Broken button). When a
|
||||||
|
|||||||
@@ -16,10 +16,12 @@ services:
|
|||||||
# MEDIA_CACHE_MAX_BYTES: "10737418240" # 10 GiB, LRU eviction past this
|
# MEDIA_CACHE_MAX_BYTES: "10737418240" # 10 GiB, LRU eviction past this
|
||||||
# MEDIA_MIN_FREE_BYTES: "5368709120" # skip caching below 5 GiB free disk
|
# MEDIA_MIN_FREE_BYTES: "5368709120" # skip caching below 5 GiB free disk
|
||||||
# MEDIA_AUTO_MAX_SECONDS: "3600" # longest video auto-cached on play
|
# MEDIA_AUTO_MAX_SECONDS: "3600" # longest video auto-cached on play
|
||||||
# MEDIA_TRANSCODE: "1" # 0 disables the x264 lane
|
# MEDIA_TRANSCODE: "1" # 0 disables the compression lane
|
||||||
|
# MEDIA_CODEC: "hevc" # hevc | h264
|
||||||
# MEDIA_CRF: "28"
|
# MEDIA_CRF: "28"
|
||||||
# MEDIA_PRESET: "slow"
|
# MEDIA_PRESET: "medium"
|
||||||
# MEDIA_THREADS: "2"
|
# MEDIA_THREADS: "2"
|
||||||
|
# MEDIA_OPT_MAX_SECONDS: "3600" # don't re-encode videos longer than this
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "-sf", "http://localhost:3000/api/version"]
|
test: ["CMD", "curl", "-sf", "http://localhost:3000/api/version"]
|
||||||
interval: 30s
|
interval: 30s
|
||||||
|
|||||||
@@ -68,6 +68,19 @@ function saveDataToStorage(jsonStr) {
|
|||||||
} catch { /* ignore parse errors */ }
|
} catch { /* ignore parse errors */ }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Can this device decode the server's HEVC copies? The compression lane
|
||||||
|
// re-encodes cached videos to HEVC (8-bit, hvc1) when that is much smaller;
|
||||||
|
// the server hands those out only to clients that send hevc=1. canPlayType
|
||||||
|
// is the browser's own claim; if a copy still fails to play here, the device
|
||||||
|
// is remembered (NO_HEVC_KEY) and stops asking — it gets H.264 as before.
|
||||||
|
const NO_HEVC_KEY = 'ytpNoHevc';
|
||||||
|
function hevcCapable() {
|
||||||
|
try { if (localStorage.getItem(NO_HEVC_KEY) === '1') return false; } catch { /* storage blocked */ }
|
||||||
|
try {
|
||||||
|
return document.createElement('video').canPlayType('video/mp4; codecs="hvc1.1.6.L93.B0"') !== '';
|
||||||
|
} catch { return false; }
|
||||||
|
}
|
||||||
|
|
||||||
// OPFS bridge wrappers — return the same shape as the Tauri cache_* commands
|
// OPFS bridge wrappers — return the same shape as the Tauri cache_* commands
|
||||||
// so the rest of app.js works without changes.
|
// so the rest of app.js works without changes.
|
||||||
|
|
||||||
@@ -82,6 +95,7 @@ async function opfsDownload(videoId, { mux = false } = {}) {
|
|||||||
// (used by "Save before playing"); default saves keep the progressive
|
// (used by "Save before playing"); default saves keep the progressive
|
||||||
// stream exactly as before.
|
// stream exactly as before.
|
||||||
if (mux) params.set('mux', '1');
|
if (mux) params.set('mux', '1');
|
||||||
|
if (hevcCapable()) params.set('hevc', '1');
|
||||||
const qs = params.toString();
|
const qs = params.toString();
|
||||||
const url = `/api/download/${encodeURIComponent(videoId)}${qs ? '?' + qs : ''}`;
|
const url = `/api/download/${encodeURIComponent(videoId)}${qs ? '?' + qs : ''}`;
|
||||||
|
|
||||||
@@ -199,7 +213,7 @@ const API = {
|
|||||||
// nocache=1 skips the server's cached copy and resolves fresh YouTube
|
// nocache=1 skips the server's cached copy and resolves fresh YouTube
|
||||||
// streams (used when the cached copy won't play on this device).
|
// streams (used when the cached copy won't play on this device).
|
||||||
getStreams: (videoId, { nocache = false } = {}) => WEB
|
getStreams: (videoId, { nocache = false } = {}) => WEB
|
||||||
? webFetch(`/api/streams?v=${encodeURIComponent(videoId)}${nocache ? '&nocache=1' : ''}`)
|
? webFetch(`/api/streams?v=${encodeURIComponent(videoId)}${nocache ? '&nocache=1' : ''}${!nocache && hevcCapable() ? '&hevc=1' : ''}`)
|
||||||
: call('yt.streams', 'yt_streams', { videoId }),
|
: call('yt.streams', 'yt_streams', { videoId }),
|
||||||
loadData: () => WEB
|
loadData: () => WEB
|
||||||
? Promise.resolve(loadDataFromStorage())
|
? Promise.resolve(loadDataFromStorage())
|
||||||
@@ -1581,6 +1595,11 @@ const Player = {
|
|||||||
// device, so the ⚠ Broken button is how a user flags a bad server copy.
|
// device, so the ⚠ Broken button is how a user flags a bad server copy.
|
||||||
if (current && current.serverCached) {
|
if (current && current.serverCached) {
|
||||||
const meta = current.meta;
|
const meta = current.meta;
|
||||||
|
// An HEVC copy that won't play means canPlayType over-promised on this
|
||||||
|
// device: stop asking for HEVC here (the server re-checks its copy).
|
||||||
|
if ((current.qualities || []).some((q) => q.codec === 'hevc')) {
|
||||||
|
try { localStorage.setItem(NO_HEVC_KEY, '1'); } catch { /* storage blocked */ }
|
||||||
|
}
|
||||||
current.serverCached = false;
|
current.serverCached = false;
|
||||||
toast('Server copy failed — streaming from YouTube. Tap ⚠ Broken to re-download it.', { duration: 5000 });
|
toast('Server copy failed — streaming from YouTube. Tap ⚠ Broken to re-download it.', { duration: 5000 });
|
||||||
this.loadVideo(meta, { preferStream: true, nocache: true, resume: this._resumeOnLoad, reveal: this._revealOnLoad });
|
this.loadVideo(meta, { preferStream: true, nocache: true, resume: this._resumeOnLoad, reveal: this._revealOnLoad });
|
||||||
|
|||||||
@@ -176,13 +176,14 @@ export async function mediaStats() {
|
|||||||
SUM(CASE WHEN status = 'ready' THEN size ELSE 0 END) AS bytes,
|
SUM(CASE WHEN status = 'ready' THEN size ELSE 0 END) AS bytes,
|
||||||
SUM(CASE WHEN status IN ('queued','downloading','validating') THEN 1 ELSE 0 END) AS pending,
|
SUM(CASE WHEN status IN ('queued','downloading','validating') THEN 1 ELSE 0 END) AS pending,
|
||||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed,
|
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failed,
|
||||||
SUM(CASE WHEN status = 'ready' AND optimized = 1 THEN 1 ELSE 0 END) AS optimized
|
SUM(CASE WHEN status = 'ready' AND optimized > 0 THEN 1 ELSE 0 END) AS optimized,
|
||||||
|
SUM(CASE WHEN status = 'ready' AND vcodec = 'hevc' THEN 1 ELSE 0 END) AS hevc
|
||||||
FROM media_cache`,
|
FROM media_cache`,
|
||||||
);
|
);
|
||||||
const o = r.rows[0] ? rowToObj(r, r.rows[0]) : {};
|
const o = r.rows[0] ? rowToObj(r, r.rows[0]) : {};
|
||||||
return {
|
return {
|
||||||
count: o.count || 0, bytes: o.bytes || 0, pending: o.pending || 0,
|
count: o.count || 0, bytes: o.bytes || 0, pending: o.pending || 0,
|
||||||
failed: o.failed || 0, optimized: o.optimized || 0,
|
failed: o.failed || 0, optimized: o.optimized || 0, hevc: o.hevc || 0,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -28,6 +28,11 @@ import { join } from 'node:path';
|
|||||||
export const HIGH = 0; // explicit save / Broken re-download
|
export const HIGH = 0; // explicit save / Broken re-download
|
||||||
export const LOW = 1; // auto-cache after a play
|
export const LOW = 1; // auto-cache after a play
|
||||||
|
|
||||||
|
// Compression-lane revision. `optimized` stores the revision a copy was last
|
||||||
|
// evaluated with; bumping this re-queues every cached copy once (rev 1 was
|
||||||
|
// x264, which never beat YouTube's own encode; rev 2 is HEVC).
|
||||||
|
export const OPT_REV = 2;
|
||||||
|
|
||||||
const ID_RE = /^[A-Za-z0-9_-]{6,64}$/;
|
const ID_RE = /^[A-Za-z0-9_-]{6,64}$/;
|
||||||
export const isMediaId = (id) => typeof id === 'string' && ID_RE.test(id);
|
export const isMediaId = (id) => typeof id === 'string' && ID_RE.test(id);
|
||||||
|
|
||||||
@@ -36,7 +41,11 @@ const OLD_GEN_GRACE_MS = 30 * 60_000; // keep superseded files for in-flight
|
|||||||
const EVICT_PROTECT_MS = 10 * 60_000; // never evict something played this recently
|
const EVICT_PROTECT_MS = 10 * 60_000; // never evict something played this recently
|
||||||
const TOUCH_DEBOUNCE_MS = 60_000;
|
const TOUCH_DEBOUNCE_MS = 60_000;
|
||||||
const EST_BYTES_PER_SEC = 250 * 1024; // ~2 Mbps: pre-compression 720p + 128k AAC
|
const EST_BYTES_PER_SEC = 250 * 1024; // ~2 Mbps: pre-compression 720p + 128k AAC
|
||||||
const MIN_SAVING = 0.15; // keep an x264 re-encode only if ≥15% smaller
|
const MIN_SAVING = 0.15; // keep a re-encode only if ≥15% smaller
|
||||||
|
// x265 tuning from the codec benchmark: stronger adaptive quantisation and
|
||||||
|
// psy settings, no SAO blur, lighter deblock, longer lookahead.
|
||||||
|
const X265_TUNING = 'aq-mode=3:aq-strength=0.8:psy-rd=1.5:psy-rdoq=1.0:no-sao=1:deblock=-1,-1:'
|
||||||
|
+ 'bframes=8:rc-lookahead=60:ref=6:no-strong-intra-smoothing=1';
|
||||||
const DAY_MS = 24 * 3600_000;
|
const DAY_MS = 24 * 3600_000;
|
||||||
|
|
||||||
// A condition that means "don't cache this right now, just stream it" — not a
|
// A condition that means "don't cache this right now, just stream it" — not a
|
||||||
@@ -69,20 +78,22 @@ function run(cmd, args) {
|
|||||||
// ----------------------------------------------------------------------------
|
// ----------------------------------------------------------------------------
|
||||||
// Validation gate. Throws with a reason; resolves to probe facts on success.
|
// Validation gate. Throws with a reason; resolves to probe facts on success.
|
||||||
// 1. file exists and is not tiny
|
// 1. file exists and is not tiny
|
||||||
// 2. exactly one H.264 8-bit 4:2:0 video stream + an AAC audio stream
|
// 2. exactly one 8-bit 4:2:0 video stream in an allowed codec + an AAC
|
||||||
// (AV1/VP9/10-bit play on almost nothing Apple — see SAVE_MUX_FORMAT)
|
// audio stream. Downloads must be H.264 (AV1/VP9 play on almost nothing
|
||||||
|
// Apple — see SAVE_MUX_FORMAT); the compression lane may produce HEVC,
|
||||||
|
// which must carry the `hvc1` tag or Safari refuses it.
|
||||||
// 3. container AND per-stream durations cover the expected length
|
// 3. container AND per-stream durations cover the expected length
|
||||||
// 4. full demux pass reports no errors (catches corrupt/truncated boxes)
|
// 4. full demux pass reports no errors (catches corrupt/truncated boxes)
|
||||||
// 5. the first and last seconds actually decode
|
// 5. the first and last seconds actually decode
|
||||||
// ----------------------------------------------------------------------------
|
// ----------------------------------------------------------------------------
|
||||||
export async function validateMedia(path, expected, { ffmpeg = 'ffmpeg', ffprobe = 'ffprobe' } = {}) {
|
export async function validateMedia(path, expected, { ffmpeg = 'ffmpeg', ffprobe = 'ffprobe', codecs = ['h264', 'hevc'] } = {}) {
|
||||||
const size = sizeOf(path);
|
const size = sizeOf(path);
|
||||||
if (!size) throw new Error('validation: output file missing');
|
if (!size) throw new Error('validation: output file missing');
|
||||||
if (size < MIN_BYTES) throw new Error(`validation: file too small (${size} bytes)`);
|
if (size < MIN_BYTES) throw new Error(`validation: file too small (${size} bytes)`);
|
||||||
|
|
||||||
const pr = await run(ffprobe, [
|
const pr = await run(ffprobe, [
|
||||||
'-v', 'error',
|
'-v', 'error',
|
||||||
'-show_entries', 'format=duration:stream=codec_type,codec_name,height,pix_fmt,duration:stream_disposition=attached_pic',
|
'-show_entries', 'format=duration:stream=codec_type,codec_name,codec_tag_string,height,pix_fmt,duration:stream_disposition=attached_pic',
|
||||||
'-of', 'json', path,
|
'-of', 'json', path,
|
||||||
]);
|
]);
|
||||||
if (pr.code !== 0) throw new Error('validation: ffprobe failed: ' + tail(pr.stderr));
|
if (pr.code !== 0) throw new Error('validation: ffprobe failed: ' + tail(pr.stderr));
|
||||||
@@ -93,7 +104,10 @@ export async function validateMedia(path, expected, { ffmpeg = 'ffmpeg', ffprobe
|
|||||||
const audio = streams.filter((s) => s.codec_type === 'audio');
|
const audio = streams.filter((s) => s.codec_type === 'audio');
|
||||||
if (video.length !== 1) throw new Error(`validation: expected 1 video stream, found ${video.length}`);
|
if (video.length !== 1) throw new Error(`validation: expected 1 video stream, found ${video.length}`);
|
||||||
const v = video[0];
|
const v = video[0];
|
||||||
if (v.codec_name !== 'h264') throw new Error(`validation: video codec ${v.codec_name} is not h264`);
|
if (!codecs.includes(v.codec_name)) throw new Error(`validation: video codec ${v.codec_name} is not ${codecs.join('/')}`);
|
||||||
|
if (v.codec_name === 'hevc' && v.codec_tag_string !== 'hvc1') {
|
||||||
|
throw new Error(`validation: HEVC tagged ${v.codec_tag_string} (Safari needs hvc1)`);
|
||||||
|
}
|
||||||
if (v.pix_fmt && !/^yuvj?420p$/.test(v.pix_fmt)) throw new Error(`validation: pixel format ${v.pix_fmt} is not 8-bit 4:2:0`);
|
if (v.pix_fmt && !/^yuvj?420p$/.test(v.pix_fmt)) throw new Error(`validation: pixel format ${v.pix_fmt} is not 8-bit 4:2:0`);
|
||||||
const a = audio.find((s) => s.codec_name === 'aac');
|
const a = audio.find((s) => s.codec_name === 'aac');
|
||||||
if (!a) throw new Error('validation: no AAC audio stream');
|
if (!a) throw new Error('validation: no AAC audio stream');
|
||||||
@@ -147,7 +161,7 @@ export function createMediaCache({
|
|||||||
minFreeBytes = 5 * 1024 ** 3,
|
minFreeBytes = 5 * 1024 ** 3,
|
||||||
autoMaxSeconds = 3600,
|
autoMaxSeconds = 3600,
|
||||||
saveMaxSeconds = 3 * 3600,
|
saveMaxSeconds = 3 * 3600,
|
||||||
transcode = { enabled: true, crf: 28, preset: 'slow', threads: 2 },
|
transcode = { enabled: true, codec: 'hevc', crf: 28, preset: 'medium', threads: 2, maxSeconds: 3600 },
|
||||||
freeBytes = () => { const s = statfsSync(dir); return s.bavail * s.bsize; },
|
freeBytes = () => { const s = statfsSync(dir); return s.bavail * s.bsize; },
|
||||||
now = () => Date.now(),
|
now = () => Date.now(),
|
||||||
log = console,
|
log = console,
|
||||||
@@ -285,7 +299,7 @@ export function createMediaCache({
|
|||||||
const rm = await run(ffmpeg, ['-v', 'error', '-nostdin', '-y', '-i', raw,
|
const rm = await run(ffmpeg, ['-v', 'error', '-nostdin', '-y', '-i', raw,
|
||||||
'-map', '0:v:0', '-map', '0:a:0', '-c', 'copy', '-movflags', '+faststart', '-f', 'mp4', norm]);
|
'-map', '0:v:0', '-map', '0:a:0', '-c', 'copy', '-movflags', '+faststart', '-f', 'mp4', norm]);
|
||||||
if (rm.code !== 0) throw new Error('remux failed: ' + tail(rm.stderr));
|
if (rm.code !== 0) throw new Error('remux failed: ' + tail(rm.stderr));
|
||||||
const probe = await validateMedia(norm, expected, { ffmpeg, ffprobe });
|
const probe = await validateMedia(norm, expected, { ffmpeg, ffprobe, codecs: ['h264'] });
|
||||||
|
|
||||||
const m4a = `${base}.norm.m4a`;
|
const m4a = `${base}.norm.m4a`;
|
||||||
const ex = await run(ffmpeg, ['-v', 'error', '-nostdin', '-y', '-i', norm,
|
const ex = await run(ffmpeg, ['-v', 'error', '-nostdin', '-y', '-i', norm,
|
||||||
@@ -332,11 +346,15 @@ export function createMediaCache({
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ---- Compression lane ----------------------------------------------------
|
// ---- Compression lane ----------------------------------------------------
|
||||||
// CRF (quality-targeted, not a fixed bitrate) at a slow preset is where
|
// Re-encodes a ready copy with CRF (quality-targeted) and keeps the result
|
||||||
// small files come from — static lyric/slide videos collapse to a fraction
|
// only when it is ≥15% smaller. Measured 2026-09-13: x264 can't beat
|
||||||
// of YouTube's bitrate. 60 fps is capped to 30. Audio is copied untouched.
|
// YouTube's own H.264 (it needed MORE bitrate for the same VMAF), while
|
||||||
// Output stays 8-bit H.264: 10-bit / HEVC / AV1 are smaller but do not
|
// x265 8-bit medium CRF 28 cut live worship video ~44% (VMAF ~93) at ~2x
|
||||||
// play in iOS Safari's <video>.
|
// realtime on the homelab's 2 niced threads. HEVC plays on iPhone Safari,
|
||||||
|
// Android and Chrome/Edge with hardware decode; the server only serves an
|
||||||
|
// HEVC copy to clients that ask for it (?hevc=1), everyone else streams.
|
||||||
|
// 8-bit on purpose — the widest hardware-decode support. 60 fps is capped
|
||||||
|
// to 30; audio is copied untouched.
|
||||||
function enqueueOptimize(id) {
|
function enqueueOptimize(id) {
|
||||||
if (!transcode || !transcode.enabled || optPending.has(id)) return;
|
if (!transcode || !transcode.enabled || optPending.has(id)) return;
|
||||||
optPending.add(id);
|
optPending.add(id);
|
||||||
@@ -355,33 +373,41 @@ export function createMediaCache({
|
|||||||
|
|
||||||
async function runOptimize(id) {
|
async function runOptimize(id) {
|
||||||
const row = await db.getMedia(id);
|
const row = await db.getMedia(id);
|
||||||
if (!row || row.status !== 'ready' || row.optimized) return;
|
if (!row || row.status !== 'ready' || (row.optimized || 0) >= OPT_REV) return;
|
||||||
const src = fileFor(id, row.gen);
|
const src = fileFor(id, row.gen);
|
||||||
if (!existsSync(src)) return;
|
if (!existsSync(src)) return;
|
||||||
const prefix = `${id}-${now()}-x264`;
|
|
||||||
const out = join(tmpDir, `${prefix}.mp4`);
|
|
||||||
const markDone = async () => {
|
const markDone = async () => {
|
||||||
const cur = await db.getMedia(id);
|
const cur = await db.getMedia(id);
|
||||||
if (cur && cur.status === 'ready' && cur.gen === row.gen) await db.upsertMedia(id, { optimized: 1 });
|
if (cur && cur.status === 'ready' && cur.gen === row.gen) await db.upsertMedia(id, { optimized: OPT_REV });
|
||||||
};
|
};
|
||||||
|
// Very long videos would tie the lane up for hours on 2 niced threads.
|
||||||
|
if (transcode.maxSeconds && row.duration > transcode.maxSeconds) { await markDone(); return; }
|
||||||
|
const hevc = transcode.codec !== 'h264';
|
||||||
|
const prefix = `${id}-${now()}-${hevc ? 'x265' : 'x264'}`;
|
||||||
|
const out = join(tmpDir, `${prefix}.mp4`);
|
||||||
try {
|
try {
|
||||||
|
const threads = String(transcode.threads);
|
||||||
|
const video = hevc
|
||||||
|
? ['-c:v', 'libx265', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
|
||||||
|
'-pix_fmt', 'yuv420p', '-tag:v', 'hvc1',
|
||||||
|
'-x265-params', `log-level=error:pools=${threads}:${X265_TUNING}`]
|
||||||
|
: ['-c:v', 'libx264', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
|
||||||
|
'-profile:v', 'high', '-level:v', '4.0', '-pix_fmt', 'yuv420p', '-threads', threads];
|
||||||
const args = ['-v', 'error', '-nostdin', '-y', '-i', src,
|
const args = ['-v', 'error', '-nostdin', '-y', '-i', src,
|
||||||
'-map', '0:v:0', '-map', '0:a:0',
|
'-map', '0:v:0', '-map', '0:a:0',
|
||||||
'-vf', 'scale=-2:min(720\\,ih)',
|
'-vf', 'scale=-2:min(720\\,ih)',
|
||||||
'-c:v', 'libx264', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
|
...video, '-fpsmax', '30',
|
||||||
'-profile:v', 'high', '-level:v', '4.0', '-pix_fmt', 'yuv420p',
|
|
||||||
'-threads', String(transcode.threads), '-fpsmax', '30',
|
|
||||||
'-c:a', 'copy', '-movflags', '+faststart', '-f', 'mp4', out];
|
'-c:a', 'copy', '-movflags', '+faststart', '-f', 'mp4', out];
|
||||||
const r = nice ? await run(nice, ['-n', '19', ffmpeg, ...args]) : await run(ffmpeg, args);
|
const r = nice ? await run(nice, ['-n', '19', ffmpeg, ...args]) : await run(ffmpeg, args);
|
||||||
if (r.code !== 0) throw new Error('x264 encode failed: ' + tail(r.stderr));
|
if (r.code !== 0) throw new Error(`${hevc ? 'x265' : 'x264'} encode failed: ` + tail(r.stderr));
|
||||||
const probe = await validateMedia(out, row.duration, { ffmpeg, ffprobe });
|
const probe = await validateMedia(out, row.duration, { ffmpeg, ffprobe, codecs: [hevc ? 'hevc' : 'h264'] });
|
||||||
|
|
||||||
const cur = await db.getMedia(id);
|
const cur = await db.getMedia(id);
|
||||||
if (!cur || cur.status !== 'ready' || cur.gen !== row.gen) return; // replaced or evicted meanwhile
|
if (!cur || cur.status !== 'ready' || cur.gen !== row.gen) return; // replaced or evicted meanwhile
|
||||||
const oldSize = sizeOf(src);
|
const oldSize = sizeOf(src);
|
||||||
if (probe.size > oldSize * (1 - MIN_SAVING)) {
|
if (probe.size > oldSize * (1 - MIN_SAVING)) {
|
||||||
await db.upsertMedia(id, { optimized: 1 });
|
await db.upsertMedia(id, { optimized: OPT_REV });
|
||||||
log.info?.(`[media] kept original ${id}: x264 ${fmtMB(probe.size)} vs ${fmtMB(oldSize)}`);
|
log.info?.(`[media] kept original ${id}: ${probe.vcodec} ${fmtMB(probe.size)} vs ${fmtMB(oldSize)}`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const gen = row.gen + 1;
|
const gen = row.gen + 1;
|
||||||
@@ -390,13 +416,13 @@ export function createMediaCache({
|
|||||||
const newAudio = fileFor(id, gen, 'm4a');
|
const newAudio = fileFor(id, gen, 'm4a');
|
||||||
try { linkSync(oldAudio, newAudio); } catch { copyFileSync(oldAudio, newAudio); }
|
try { linkSync(oldAudio, newAudio); } catch { copyFileSync(oldAudio, newAudio); }
|
||||||
await db.upsertMedia(id, {
|
await db.upsertMedia(id, {
|
||||||
gen, size: probe.size + sizeOf(newAudio), height: probe.height,
|
gen, size: probe.size + sizeOf(newAudio), height: probe.height, vcodec: probe.vcodec,
|
||||||
duration: probe.duration, optimized: 1, updated_at: now(),
|
duration: probe.duration, optimized: OPT_REV, updated_at: now(),
|
||||||
});
|
});
|
||||||
removeFiles(id, gen, OLD_GEN_GRACE_MS);
|
removeFiles(id, gen, OLD_GEN_GRACE_MS);
|
||||||
log.info?.(`[media] optimized ${id}: ${fmtMB(oldSize)} → ${fmtMB(probe.size)}`);
|
log.info?.(`[media] optimized ${id}: ${fmtMB(oldSize)} → ${fmtMB(probe.size)} (${probe.vcodec})`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
await markDone().catch(() => {}); // don't retry a source x264 can't improve
|
await markDone().catch(() => {}); // don't retry a source the encoder can't improve
|
||||||
throw err;
|
throw err;
|
||||||
} finally {
|
} finally {
|
||||||
sweepTmp(prefix);
|
sweepTmp(prefix);
|
||||||
@@ -505,7 +531,8 @@ export function createMediaCache({
|
|||||||
status: job ? job.status : row ? row.status : 'none',
|
status: job ? job.status : row ? row.status : 'none',
|
||||||
size: row ? row.size : 0,
|
size: row ? row.size : 0,
|
||||||
height: row ? row.height : 0,
|
height: row ? row.height : 0,
|
||||||
optimized: !!(row && row.optimized),
|
vcodec: row ? row.vcodec : null,
|
||||||
|
optimized: !!(row && (row.optimized || 0) >= OPT_REV),
|
||||||
optimizing: optActive === id,
|
optimizing: optActive === id,
|
||||||
error: row && row.status === 'failed' ? row.error : null,
|
error: row && row.status === 'failed' ? row.error : null,
|
||||||
retryAt: row && row.status === 'failed' ? row.retry_at : 0,
|
retryAt: row && row.status === 'failed' ? row.retry_at : 0,
|
||||||
@@ -542,7 +569,7 @@ export function createMediaCache({
|
|||||||
for (const r of rows) {
|
for (const r of rows) {
|
||||||
if (r.status === 'ready') {
|
if (r.status === 'ready') {
|
||||||
if (!readyFilesExist(r)) { await db.deleteMedia(r.video_id); removeFiles(r.video_id); continue; }
|
if (!readyFilesExist(r)) { await db.deleteMedia(r.video_id); removeFiles(r.video_id); continue; }
|
||||||
if (!r.optimized) enqueueOptimize(r.video_id);
|
if ((r.optimized || 0) < OPT_REV) enqueueOptimize(r.video_id);
|
||||||
} else if (r.status === 'queued' || r.status === 'downloading' || r.status === 'validating') {
|
} else if (r.status === 'queued' || r.status === 'downloading' || r.status === 'validating') {
|
||||||
startJob(r.video_id, r.priority === HIGH ? HIGH : LOW, !!r.auto, r);
|
startJob(r.video_id, r.priority === HIGH ? HIGH : LOW, !!r.auto, r);
|
||||||
resumed++;
|
resumed++;
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import { join } from 'node:path';
|
|||||||
const root = mkdtempSync(join(tmpdir(), 'ytp-media-test-'));
|
const root = mkdtempSync(join(tmpdir(), 'ytp-media-test-'));
|
||||||
process.env.DB_PATH = join(root, 'test.db');
|
process.env.DB_PATH = join(root, 'test.db');
|
||||||
const dbmod = await import('./db.js');
|
const dbmod = await import('./db.js');
|
||||||
const { createMediaCache, validateMedia, MediaSkip, HIGH, LOW } = await import('./media-cache.js');
|
const { createMediaCache, validateMedia, MediaSkip, HIGH, LOW, OPT_REV } = await import('./media-cache.js');
|
||||||
|
|
||||||
const fx = (name) => join(root, name);
|
const fx = (name) => join(root, name);
|
||||||
const ff = (...args) => {
|
const ff = (...args) => {
|
||||||
@@ -33,6 +33,11 @@ beforeAll(async () => {
|
|||||||
'-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p', fx('noaudio.mp4'));
|
'-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p', fx('noaudio.mp4'));
|
||||||
ff('-f', 'lavfi', '-i', 'testsrc=size=640x360:rate=30:duration=10', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=10',
|
ff('-f', 'lavfi', '-i', 'testsrc=size=640x360:rate=30:duration=10', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=10',
|
||||||
'-c:v', 'mpeg4', '-c:a', 'aac', '-shortest', fx('mpeg4.mp4'));
|
'-c:v', 'mpeg4', '-c:a', 'aac', '-shortest', fx('mpeg4.mp4'));
|
||||||
|
// HEVC as the compression lane writes it (hvc1) and as Safari rejects it (hev1).
|
||||||
|
ff('-f', 'lavfi', '-i', 'testsrc=size=640x360:rate=30:duration=10', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=10',
|
||||||
|
'-c:v', 'libx265', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p', '-tag:v', 'hvc1', '-x265-params', 'log-level=error',
|
||||||
|
'-c:a', 'aac', '-shortest', fx('hevc.mp4'));
|
||||||
|
ff('-i', fx('hevc.mp4'), '-c', 'copy', '-tag:v', 'hev1', fx('hevc-hev1.mp4'));
|
||||||
const buf = readFileSync(fx('good.mp4'));
|
const buf = readFileSync(fx('good.mp4'));
|
||||||
writeFileSync(fx('truncated.mp4'), buf.subarray(0, Math.floor(buf.length * 0.6)));
|
writeFileSync(fx('truncated.mp4'), buf.subarray(0, Math.floor(buf.length * 0.6)));
|
||||||
// The nasty case: moov up front (index says 10 s) but the media data is cut.
|
// The nasty case: moov up front (index says 10 s) but the media data is cut.
|
||||||
@@ -61,6 +66,14 @@ describe('validateMedia', () => {
|
|||||||
test('rejects a non-h264 file', async () => {
|
test('rejects a non-h264 file', async () => {
|
||||||
await expect(validateMedia(fx('mpeg4.mp4'), 10)).rejects.toThrow(/not h264/);
|
await expect(validateMedia(fx('mpeg4.mp4'), 10)).rejects.toThrow(/not h264/);
|
||||||
});
|
});
|
||||||
|
test('accepts hvc1-tagged HEVC for the compression lane, but not as a download', async () => {
|
||||||
|
const p = await validateMedia(fx('hevc.mp4'), 10);
|
||||||
|
expect(p.vcodec).toBe('hevc');
|
||||||
|
await expect(validateMedia(fx('hevc.mp4'), 10, { codecs: ['h264'] })).rejects.toThrow(/not h264/);
|
||||||
|
});
|
||||||
|
test('rejects hev1-tagged HEVC (Safari refuses it)', async () => {
|
||||||
|
await expect(validateMedia(fx('hevc-hev1.mp4'), 10)).rejects.toThrow(/hvc1/);
|
||||||
|
});
|
||||||
test('rejects a file shorter than the source video', async () => {
|
test('rejects a file shorter than the source video', async () => {
|
||||||
await expect(validateMedia(fx('good.mp4'), 60)).rejects.toThrow(/truncated/);
|
await expect(validateMedia(fx('good.mp4'), 60)).rejects.toThrow(/truncated/);
|
||||||
});
|
});
|
||||||
@@ -210,19 +223,39 @@ describe('media cache jobs', () => {
|
|||||||
expect(calls).toEqual(['bootAAAAAA1']);
|
expect(calls).toEqual(['bootAAAAAA1']);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('x264 compression lane swaps in a smaller copy under a new gen', async () => {
|
test('HEVC compression lane swaps in a smaller copy under a new gen', async () => {
|
||||||
await clearDb();
|
await clearDb();
|
||||||
const { cache } = makeCache({ transcode: { enabled: true, crf: 35, preset: 'ultrafast', threads: 2 } });
|
const { cache } = makeCache({ transcode: { enabled: true, codec: 'hevc', crf: 34, preset: 'ultrafast', threads: 2 } });
|
||||||
await cache.init();
|
await cache.init();
|
||||||
const first = await cache.ensureCached('optAAAAAAA1', { priority: HIGH });
|
const first = await cache.ensureCached('optAAAAAAA1', { priority: HIGH });
|
||||||
expect(await until(async () => (await dbmod.getMedia('optAAAAAAA1')).optimized === 1)).toBe(true);
|
expect(first.vcodec).toBe('h264');
|
||||||
|
expect(await until(async () => (await dbmod.getMedia('optAAAAAAA1')).optimized === OPT_REV)).toBe(true);
|
||||||
const after = await dbmod.getMedia('optAAAAAAA1');
|
const after = await dbmod.getMedia('optAAAAAAA1');
|
||||||
if (after.gen !== first.gen) {
|
expect(after.gen).toBeGreaterThan(first.gen);
|
||||||
|
expect(after.vcodec).toBe('hevc');
|
||||||
expect(after.size).toBeLessThan(first.size);
|
expect(after.size).toBeLessThan(first.size);
|
||||||
// old generation is kept for in-flight playback (grace period)
|
// old generation is kept for in-flight playback (grace period)
|
||||||
expect(await cache.filePath('optAAAAAAA1', first.gen)).not.toBeNull();
|
expect(await cache.filePath('optAAAAAAA1', first.gen)).not.toBeNull();
|
||||||
}
|
|
||||||
expect(await cache.filePath('optAAAAAAA1', after.gen, 'm4a')).not.toBeNull();
|
expect(await cache.filePath('optAAAAAAA1', after.gen, 'm4a')).not.toBeNull();
|
||||||
|
// and the HEVC copy still passes the gate verify() uses
|
||||||
|
expect(await cache.verify('optAAAAAAA1')).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('copies evaluated by an older lane revision are re-queued at boot', async () => {
|
||||||
|
await clearDb();
|
||||||
|
const seed = makeCache();
|
||||||
|
await seed.cache.init();
|
||||||
|
await seed.cache.ensureCached('revAAAAAAA1', { priority: HIGH });
|
||||||
|
await dbmod.upsertMedia('revAAAAAAA1', { optimized: 1 }); // the old x264 revision
|
||||||
|
// A fresh instance over the same dir + DB, as after a deploy.
|
||||||
|
const same = createMediaCache({
|
||||||
|
dir: seed.dir, db: { getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia,
|
||||||
|
listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia, mediaStats: dbmod.mediaStats },
|
||||||
|
log: silentLog, getInfo: async () => ({ duration: 10 }), download: async () => { throw new Error('no download expected'); },
|
||||||
|
transcode: { enabled: true, codec: 'hevc', crf: 34, preset: 'ultrafast', threads: 2 }, freeBytes: () => 100 * 1024 ** 3,
|
||||||
|
});
|
||||||
|
await same.init();
|
||||||
|
expect(await until(async () => (await dbmod.getMedia('revAAAAAAA1')).optimized === OPT_REV)).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('verify() drops and refetches a copy that went bad on disk, keeps a good one', async () => {
|
test('verify() drops and refetches a copy that went bad on disk, keeps a good one', async () => {
|
||||||
|
|||||||
@@ -506,7 +506,7 @@ app.get('/api/streams', async (c) => {
|
|||||||
if (c.req.query('nocache') !== '1') {
|
if (c.req.query('nocache') !== '1') {
|
||||||
try {
|
try {
|
||||||
const row = await media.getReady(videoId);
|
const row = await media.getReady(videoId);
|
||||||
if (row) {
|
if (servableTo(row, c.req.query('hevc') === '1')) {
|
||||||
media.touch(videoId);
|
media.touch(videoId);
|
||||||
return c.json({ ok: true, data: cachedStreamsPayload(videoId, row) });
|
return c.json({ ok: true, data: cachedStreamsPayload(videoId, row) });
|
||||||
}
|
}
|
||||||
@@ -963,15 +963,24 @@ const media = createMediaCache({
|
|||||||
saveMaxSeconds: MAX_SAVE_SECONDS,
|
saveMaxSeconds: MAX_SAVE_SECONDS,
|
||||||
transcode: {
|
transcode: {
|
||||||
enabled: process.env.MEDIA_TRANSCODE !== '0',
|
enabled: process.env.MEDIA_TRANSCODE !== '0',
|
||||||
|
codec: process.env.MEDIA_CODEC || 'hevc', // hevc | h264
|
||||||
crf: envNum('MEDIA_CRF', 28),
|
crf: envNum('MEDIA_CRF', 28),
|
||||||
preset: process.env.MEDIA_PRESET || 'slow',
|
preset: process.env.MEDIA_PRESET || 'medium',
|
||||||
threads: envNum('MEDIA_THREADS', 2),
|
threads: envNum('MEDIA_THREADS', 2),
|
||||||
|
maxSeconds: envNum('MEDIA_OPT_MAX_SECONDS', 3600),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// A cached copy the compression lane turned into HEVC is only handed to
|
||||||
|
// clients that said they can decode it (?hevc=1 — the page checks
|
||||||
|
// canPlayType for hvc1). Everyone else keeps the pre-cache behaviour.
|
||||||
|
function servableTo(row, hevcOk) {
|
||||||
|
return !!row && (row.vcodec !== 'hevc' || hevcOk);
|
||||||
|
}
|
||||||
|
|
||||||
// /api/streams response for a server-cached video: one progressive quality
|
// /api/streams response for a server-cached video: one progressive quality
|
||||||
// (the mp4 carries its own audio) plus the m4a sidecar for audio-only mode.
|
// (the mp4 carries its own audio) plus the m4a sidecar for audio-only mode.
|
||||||
// Same shape as the YouTube path; `serverCached` is additive.
|
// Same shape as the YouTube path; `serverCached` and `codec` are additive.
|
||||||
function cachedStreamsPayload(videoId, row) {
|
function cachedStreamsPayload(videoId, row) {
|
||||||
let meta = {};
|
let meta = {};
|
||||||
try { meta = JSON.parse(row.meta || '{}'); } catch { /* corrupt — use defaults */ }
|
try { meta = JSON.parse(row.meta || '{}'); } catch { /* corrupt — use defaults */ }
|
||||||
@@ -993,6 +1002,7 @@ function cachedStreamsPayload(videoId, row) {
|
|||||||
hasAudio: true,
|
hasAudio: true,
|
||||||
url,
|
url,
|
||||||
ext: 'mp4',
|
ext: 'mp4',
|
||||||
|
codec: row.vcodec || 'h264',
|
||||||
}],
|
}],
|
||||||
serverCached: true,
|
serverCached: true,
|
||||||
};
|
};
|
||||||
@@ -1117,7 +1127,10 @@ app.get('/api/download/:videoId', async (c) => {
|
|||||||
let cacheErr = null;
|
let cacheErr = null;
|
||||||
try {
|
try {
|
||||||
const row = await media.ensureCached(videoId, { priority: HIGH });
|
const row = await media.ensureCached(videoId, { priority: HIGH });
|
||||||
return cachedDownloadResponse(videoId, fp, row);
|
// A device that can't decode HEVC must not save the HEVC copy — give it
|
||||||
|
// the legacy H.264 save instead (the ?mux=1 path included).
|
||||||
|
if (servableTo(row, c.req.query('hevc') === '1')) return cachedDownloadResponse(videoId, fp, row);
|
||||||
|
cacheErr = Object.assign(new Error('cached copy is HEVC; client did not ask for it'), { code: 'SKIPPED' });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
cacheErr = err;
|
cacheErr = err;
|
||||||
console.warn(`[ytplayer] cache save unavailable for ${videoId} (${err.code || 'error'}): ${err.message}`);
|
console.warn(`[ytplayer] cache save unavailable for ${videoId} (${err.code || 'error'}): ${err.message}`);
|
||||||
|
|||||||
Reference in New Issue
Block a user