Sample two short windows before re-encoding a cached video and skip the full encode when HEVC will not save 15 percent

CLAUDE.md

@@ -44,11 +44,16 @@ JSON shapes mirror the Tauri Rust bridge exactly — don't change one side alone
   different bytes. A replaced gen is kept 30 min so in-flight Range playback survives.
 - **Compression lane** (rev `OPT_REV` = 2): niced x265 8-bit `-preset medium -crf 28`,
   `hvc1` tag, anime-style tuning (`X265_TUNING`), `-fpsmax 30`, audio copied — kept only
-  if ≥15% smaller, and the copy's `vcodec` becomes `hevc`. Benchmarked 2026-09-13 (VMAF,
+  if ≥15% smaller, and the copy's `vcodec` becomes `hevc`. **Sample-first**: videos ≥60 s
-  codec-test page in Windows Docs): x264 can NOT beat YouTube's own H.264 (needs more
+  get two 10-s windows encoded first and compared with the source's own packet bytes
-  bitrate at equal quality); x265 cuts live worship video ~35–44% at ~2× realtime on the
+  over the same timestamps; the full encode only runs if the prediction clears the 15%
-  homelab's 2 threads; lyric videos (~47 kbps video, mostly audio) keep the original.
+  bar (a skip costs ~20 s of video; a full encode ~2× realtime on the homelab).
-  Bumping `OPT_REV` re-queues every cached copy once at boot.
+  **Why:** re-encoding YouTube's already-compressed H.264 usually needs MORE bits for
+  LOWER quality (generation loss) — on prod 0 of 5 videos shrank at CRF 28, one grew 25%
+  (VMAF 93.6 at 126% of source; capping to 60–70% drops VMAF to ~84–87). Only videos
+  where YouTube over-spent bits win (a live worship video: −34%). AnimeOut-style sizes
+  come from pristine Blu-ray sources, which YouTube never gives us. Bumping `OPT_REV`
+  re-queues every cached copy once at boot.
 - **HEVC gating**: an HEVC copy is served only to clients that send `?hevc=1` on
   `/api/streams` and `/api/download` (app.js `hevcCapable()` = canPlayType hvc1, minus a
   per-device `ytpNoHevc` flag set when an HEVC copy fails to play). Everyone else gets

server/media-cache.js

@@ -42,6 +42,8 @@ const EVICT_PROTECT_MS  = 10 * 60_000;   // never evict something played this re
 const TOUCH_DEBOUNCE_MS = 60_000;
 const EST_BYTES_PER_SEC = 250 * 1024;    // ~2 Mbps: pre-compression 720p + 128k AAC
 const MIN_SAVING        = 0.15;          // keep a re-encode only if ≥15% smaller
+const SAMPLE_SECONDS    = 10;            // per sample window (two windows)
+const SAMPLE_MIN_DURATION = 60;          // shorter videos are cheap enough to encode whole
 // x265 tuning from the codec benchmark: stronger adaptive quantisation and
 // psy settings, no SAO blur, lighter deblock, longer lookahead.
 const X265_TUNING = 'aq-mode=3:aq-strength=0.8:psy-rd=1.5:psy-rdoq=1.0:no-sao=1:deblock=-1,-1:'
@@ -385,14 +387,29 @@ export function createMediaCache({
     const hevc = transcode.codec !== 'h264';
     const prefix = `${id}-${now()}-${hevc ? 'x265' : 'x264'}`;
     const out = join(tmpDir, `${prefix}.mp4`);
+    const threads = String(transcode.threads);
+    const video = hevc
+      ? ['-c:v', 'libx265', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
+         '-pix_fmt', 'yuv420p', '-tag:v', 'hvc1',
+         '-x265-params', `log-level=error:pools=${threads}:${X265_TUNING}`]
+      : ['-c:v', 'libx264', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
+         '-profile:v', 'high', '-level:v', '4.0', '-pix_fmt', 'yuv420p', '-threads', threads];
     try {
-      const threads = String(transcode.threads);
+      // Sample first. Re-encoding YouTube's already-compressed H.264 usually
-      const video = hevc
+      // needs MORE bits than YouTube spent (measured on prod: 0 of 5 videos
-        ? ['-c:v', 'libx265', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
+      // shrank at CRF 28; one grew 25%), so spend ~20 s of video finding out
-           '-pix_fmt', 'yuv420p', '-tag:v', 'hvc1',
+      // before ~2x realtime of CPU on the whole thing.
-           '-x265-params', `log-level=error:pools=${threads}:${X265_TUNING}`]
+      if (row.duration >= SAMPLE_MIN_DURATION) {
-        : ['-c:v', 'libx264', '-preset', String(transcode.preset), '-crf', String(transcode.crf),
+        const ratio = await sampleRatio(src, row.duration, video, prefix);
-           '-profile:v', 'high', '-level:v', '4.0', '-pix_fmt', 'yuv420p', '-threads', threads];
+        const mp4 = sizeOf(src);
+        const videoBytes = Math.max(0, mp4 - sizeOf(fileFor(id, row.gen, 'm4a')));
+        const predicted = mp4 - videoBytes * (1 - ratio);
+        if (predicted > mp4 * (1 - MIN_SAVING)) {
+          await markDone();
+          log.info?.(`[media] kept original ${id}: sample predicts ${Math.round(predicted / mp4 * 100)}% of ${fmtMB(mp4)}`);
+          return;
+        }
+      }
       const args = ['-v', 'error', '-nostdin', '-y', '-i', src,
         '-map', '0:v:0', '-map', '0:a:0',
         '-vf', 'scale=-2:min(720\\,ih)',
@@ -429,6 +446,34 @@ export function createMediaCache({
     }
   }
 
+  // Encode two short windows (at 25% and 60% of the video) and compare them
+  // with the source's own video bytes over exactly the same timestamps.
+  // Returns encoded/source (below 1 = smaller).
+  async function sampleRatio(src, duration, videoArgs, prefix) {
+    let srcBytes = 0;
+    let encBytes = 0;
+    const starts = [0.25, 0.6].map((f) => Math.floor(duration * f));
+    for (const [i, start] of starts.entries()) {
+      const end = start + SAMPLE_SECONDS;
+      const pr = await run(ffprobe, ['-v', 'error', '-select_streams', 'v:0',
+        '-read_intervals', `${Math.max(0, start - 10)}%${end + 1}`,
+        '-show_entries', 'packet=pts_time,size', '-of', 'csv=p=0', src]);
+      if (pr.code !== 0) throw new Error('sample probe failed: ' + tail(pr.stderr));
+      for (const line of pr.stdout.split('\n')) {
+        const [pts, size] = line.split(',');
+        const t = parseFloat(pts);
+        if (t >= start && t < end) srcBytes += parseInt(size, 10) || 0;
+      }
+      const out = join(tmpDir, `${prefix}.sample${i}.mp4`);
+      const args = ['-v', 'error', '-nostdin', '-y', '-ss', String(start), '-t', String(SAMPLE_SECONDS), '-i', src,
+        '-map', '0:v:0', '-an', '-vf', 'scale=-2:min(720\\,ih)', ...videoArgs, '-fpsmax', '30', '-f', 'mp4', out];
+      const r = nice ? await run(nice, ['-n', '19', ffmpeg, ...args]) : await run(ffmpeg, args);
+      if (r.code !== 0) throw new Error('sample encode failed: ' + tail(r.stderr));
+      encBytes += sizeOf(out);
+    }
+    return srcBytes > 0 ? encBytes / srcBytes : 1;
+  }
+
   // ---- Public API ----------------------------------------------------------
 
   // Resolve to the ready row, fetching it first if needed. Rejects with a

server/media-cache.test.js

@@ -38,6 +38,13 @@ beforeAll(async () => {
      '-c:v', 'libx265', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p', '-tag:v', 'hvc1', '-x265-params', 'log-level=error',
      '-c:a', 'aac', '-shortest', fx('hevc.mp4'));
   ff('-i', fx('hevc.mp4'), '-c', 'copy', '-tag:v', 'hev1', fx('hevc-hev1.mp4'));
+  // 64 s sources for the sample-first path (videos ≥ 60 s get sampled):
+  // "fat" = far more bits than needed (HEVC will win), "lean" = already
+  // starved (HEVC at CRF 28 can't beat it — like most YouTube encodes).
+  ff('-f', 'lavfi', '-i', 'testsrc2=size=640x360:rate=24:duration=64', '-f', 'lavfi', '-i', 'sine=frequency=330:duration=64',
+     '-c:v', 'libx264', '-preset', 'ultrafast', '-qp', '8', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-shortest', fx('fat64.mp4'));
+  ff('-f', 'lavfi', '-i', 'testsrc2=size=640x360:rate=24:duration=64', '-f', 'lavfi', '-i', 'sine=frequency=330:duration=64',
+     '-c:v', 'libx264', '-preset', 'medium', '-crf', '44', '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-shortest', fx('lean64.mp4'));
   const buf = readFileSync(fx('good.mp4'));
   writeFileSync(fx('truncated.mp4'), buf.subarray(0, Math.floor(buf.length * 0.6)));
   // The nasty case: moov up front (index says 10 s) but the media data is cut.
@@ -241,6 +248,34 @@ describe('media cache jobs', () => {
     expect(await cache.verify('optAAAAAAA1')).toBe(true);
   });
 
+  test('sample-first: a source HEVC cannot beat is skipped without a full encode', async () => {
+    await clearDb();
+    const lines = [];
+    const { cache } = makeCache({ fixture: 'lean64.mp4', duration: 64, log: { info: (m) => lines.push(m), warn: (m) => lines.push(m) },
+      transcode: { enabled: true, codec: 'hevc', crf: 28, preset: 'ultrafast', threads: 2 } });
+    await cache.init();
+    const first = await cache.ensureCached('sampAAAAAA1', { priority: HIGH });
+    expect(await until(async () => (await dbmod.getMedia('sampAAAAAA1')).optimized === OPT_REV, 60000)).toBe(true);
+    const after = await dbmod.getMedia('sampAAAAAA1');
+    expect(after.gen).toBe(first.gen);
+    expect(after.vcodec).toBe('h264');
+    expect(lines.some((l) => /sample predicts/.test(l))).toBe(true);
+  }, 90000);
+
+  test('sample-first: a source HEVC can beat gets the full encode and swaps', async () => {
+    await clearDb();
+    const lines = [];
+    const { cache } = makeCache({ fixture: 'fat64.mp4', duration: 64, log: { info: (m) => lines.push(m), warn: (m) => lines.push(m) },
+      transcode: { enabled: true, codec: 'hevc', crf: 28, preset: 'ultrafast', threads: 2 } });
+    await cache.init();
+    const first = await cache.ensureCached('sampAAAAAA2', { priority: HIGH });
+    expect(await until(async () => (await dbmod.getMedia('sampAAAAAA2')).optimized === OPT_REV, 60000)).toBe(true);
+    const after = await dbmod.getMedia('sampAAAAAA2');
+    expect(after.vcodec).toBe('hevc');
+    expect(after.gen).toBeGreaterThan(first.gen);
+    expect(lines.some((l) => /sample predicts/.test(l))).toBe(false);
+  }, 90000);
+
   test('copies evaluated by an older lane revision are re-queued at boot', async () => {
     await clearDb();
     const seed = makeCache();