Cache every played or saved video on the server via background jobs with a validation gate and a Broken re-download button

server/server.js

@@ -10,7 +10,11 @@
  *   GET  /api/channel?c=<channel>      yt-dlp channel uploads → slim card array
  *   GET  /api/streams?v=<videoId>      yt-dlp stream info → {meta, audioUrl, qualities} (proxied URLs)
  *   GET  /api/play?v=<id>&f=<fmt>      same-origin playback proxy (Range-aware) → media bytes
- *   GET  /api/download/:videoId        proxy best progressive stream → binary
+ *   GET  /api/download/:videoId        server-cached copy (fetched by a background job) → binary
+ *   GET  /api/media/:id?g=<gen>[&a=1]  Range-aware server-cached mp4 (or m4a audio sidecar)
+ *   GET  /api/media/:id/status         { status, size, height, optimized, error }
+ *   POST /api/media/:id/redownload     "Broken" button: drop the copy, fetch it again
+ *   GET  /api/media/stats              cache totals, budget, free disk, queue
  *   GET  /api/version                  { version }
  *   POST /api/user/sync                upsert user playlists + last-seen version
  *   GET  /api/user/data?fp=<fp>        retrieve stored playlists + history
@@ -28,11 +32,13 @@ import { serveStatic }  from 'hono/bun';
 import { logger }       from 'hono/logger';
 import { spawn }        from 'node:child_process';
 import { createServer } from 'node:http';
-import { readFileSync, readdirSync, existsSync, statSync, openSync, unlinkSync, createReadStream } from 'node:fs';
+import { readFileSync, readdirSync, existsSync, statSync, openSync, unlinkSync, createReadStream, mkdirSync } from 'node:fs';
 import { Readable }      from 'node:stream';
 import { tmpdir }        from 'node:os';
 import { createHash }    from 'node:crypto';
-import { initDb, upsertUser, recordVideoAccess, getUserData, createProfile, getProfile, saveProfile, createSharedPlaylist, getSharedPlaylist, queueInboxPlaylist, listInbox, deleteInboxItem, countInbox } from './db.js';
+import { initDb, upsertUser, recordVideoAccess, getUserData, createProfile, getProfile, saveProfile, createSharedPlaylist, getSharedPlaylist, queueInboxPlaylist, listInbox, deleteInboxItem, countInbox,
+         getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats } from './db.js';
+import { createMediaCache, HIGH, LOW } from './media-cache.js';
 
 // A media proxy must not die because one client's stream hit an edge case
 // (see /api/play cancel()): log and keep serving instead of crash-looping.
@@ -494,6 +500,24 @@ app.get('/api/streams', async (c) => {
   const videoId = (c.req.query('v') || '').replace(/[/\\:?<>|*"]/g, '').trim();
   if (!videoId) return c.json({ ok: false, error: 'missing videoId' }, 400);
 
+  // Server already holds a validated copy → answer from the DB alone, no
+  // yt-dlp round trip. ?nocache=1 (the client's fallback when a cached copy
+  // won't play on its device) forces the YouTube path below.
+  if (c.req.query('nocache') !== '1') {
+    try {
+      const row = await media.getReady(videoId);
+      if (row) {
+        media.touch(videoId);
+        return c.json({ ok: true, data: cachedStreamsPayload(videoId, row) });
+      }
+    } catch (err) {
+      console.warn(`[media] cache lookup failed for ${videoId}:`, err.message);
+    }
+  } else {
+    // A client couldn't play our copy — re-check it server-side (background).
+    media.verify(videoId).catch(() => {});
+  }
+
   try {
     const { info, formats } = await resolveStreams(videoId);
 
@@ -536,6 +560,10 @@ app.get('/api/streams', async (c) => {
       return c.json({ ok: false, error: live ? 'Live streams are not supported' : 'No playable formats for this video' }, 422);
     }
 
+    // Played but not cached yet: fetch a copy in the background so the next
+    // play is served from disk. Runs server-side; the client never waits.
+    media.ensureCached(videoId, { priority: LOW, auto: true }).catch(() => {});
+
     return c.json({
       ok: true,
       data: {
@@ -852,15 +880,16 @@ async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) {
 // concatenate them into one continuous mp4 — the user's custom cut. The result
 // is streamed to the browser exactly like a normal save, so OPFS stores it
 // under the caller-chosen custom id. Every temp file is swept afterwards.
-async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal) {
-  await assertNotLive(videoId);
+async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal, cachedSrc = null) {
+  if (!cachedSrc) await assertNotLive(videoId);
   const tmpBase = `ytp-edit-${videoId}-${Date.now()}`;
-  const srcTmp  = `${tmpdir()}/${tmpBase}.src.mp4`;
+  const srcTmp  = cachedSrc || `${tmpdir()}/${tmpBase}.src.mp4`;
   const outTmp  = `${tmpdir()}/${tmpBase}.out.mp4`;
   let size, fd;
   try {
-    // 1) Grab the full source (video+audio merged) so ffmpeg has both streams.
-    await withSaveSlot(() => runYtdlpResilient([
+    // 1) Grab the full source (video+audio merged) so ffmpeg has both streams
+    //    — unless the server cache already holds it.
+    if (!cachedSrc) await withSaveSlot(() => runYtdlpResilient([
       `https://www.youtube.com/watch?v=${videoId}`,
       '--no-warnings', '--no-playlist',
       '-f', SAVE_MUX_FORMAT,
@@ -900,10 +929,168 @@ async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal) {
   });
 }
 
+// ============================================================================
+// Server-side media cache (see media-cache.js). One validated ≤720p H.264 +
+// AAC copy per played/saved video under $MEDIA_DIR, fetched by server-owned
+// jobs that survive the client going away. Budget-bounded LRU; never expires
+// by time. Downloads still go through withSaveSlot so they stay serialized
+// with any legacy fallback save (the homelab downlink is the constraint).
+// ============================================================================
+const envNum = (k, d) => (process.env[k] !== undefined && process.env[k] !== '' ? Number(process.env[k]) : d);
+const MEDIA_DIR = process.env.MEDIA_DIR || './data/media';
+mkdirSync(MEDIA_DIR, { recursive: true });
+
+const media = createMediaCache({
+  dir: MEDIA_DIR,
+  db: { getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats },
+  getInfo: async (videoId) => (await resolveStreams(videoId)).info,
+  download: async (videoId, out) => {
+    await withSaveSlot(() => runYtdlpResilient([
+      `https://www.youtube.com/watch?v=${videoId}`,
+      '--no-warnings', '--no-playlist',
+      '-f', SAVE_MUX_FORMAT,
+      '--merge-output-format', 'mp4',
+      '--limit-rate', DOWNLOAD_RATE,
+      '-o', out,
+    ]));
+    return out;
+  },
+  ffmpeg: FFMPEG,
+  ffprobe: process.env.FFPROBE_PATH || 'ffprobe',
+  maxBytes:       envNum('MEDIA_CACHE_MAX_BYTES', 10 * 1024 ** 3),
+  minFreeBytes:   envNum('MEDIA_MIN_FREE_BYTES', 5 * 1024 ** 3),
+  autoMaxSeconds: envNum('MEDIA_AUTO_MAX_SECONDS', 3600),
+  saveMaxSeconds: MAX_SAVE_SECONDS,
+  transcode: {
+    enabled: process.env.MEDIA_TRANSCODE !== '0',
+    crf:     envNum('MEDIA_CRF', 28),
+    preset:  process.env.MEDIA_PRESET || 'slow',
+    threads: envNum('MEDIA_THREADS', 2),
+  },
+});
+
+// /api/streams response for a server-cached video: one progressive quality
+// (the mp4 carries its own audio) plus the m4a sidecar for audio-only mode.
+// Same shape as the YouTube path; `serverCached` is additive.
+function cachedStreamsPayload(videoId, row) {
+  let meta = {};
+  try { meta = JSON.parse(row.meta || '{}'); } catch { /* corrupt — use defaults */ }
+  const url = `/api/media/${videoId}?g=${row.gen}`;
+  return {
+    meta: {
+      id:         videoId,
+      title:      meta.title || '(untitled)',
+      channel:    meta.channel || '',
+      channelId:  meta.channelId || '',
+      channelUrl: meta.channelUrl || '',
+      duration:   meta.duration || Math.round(row.duration || 0),
+      thumbnail:  `https://i.ytimg.com/vi/${videoId}/hqdefault.jpg`,
+    },
+    audioUrl: `${url}&a=1`,
+    qualities: [{
+      label:    (row.height || 720) + 'p',
+      height:   row.height || 720,
+      hasAudio: true,
+      url,
+      ext:      'mp4',
+    }],
+    serverCached: true,
+  };
+}
+
+// Serve a file with byte-Range support (the <video> element seeks with it).
+function rangeFileResponse(c, path, contentType, cacheControl) {
+  const file = Bun.file(path);
+  const total = file.size;
+  const headers = {
+    'Content-Type':  contentType,
+    'Accept-Ranges': 'bytes',
+    'Cache-Control': cacheControl,
+  };
+  const range = c.req.header('range');
+  if (!range) return new Response(file, { status: 200, headers });
+  const m = /^bytes=(\d*)-(\d*)$/.exec(range.trim());
+  let start, end;
+  if (m && m[1] !== '') {
+    start = Number(m[1]);
+    end = m[2] !== '' ? Math.min(Number(m[2]), total - 1) : total - 1;
+  } else if (m && m[2] !== '') {
+    start = Math.max(0, total - Number(m[2]));
+    end = total - 1;
+  }
+  if (start === undefined || start > end || start >= total) {
+    return new Response(null, { status: 416, headers: { ...headers, 'Content-Range': `bytes */${total}` } });
+  }
+  return new Response(file.slice(start, end + 1), {
+    status: 206,
+    headers: { ...headers, 'Content-Range': `bytes ${start}-${end}/${total}` },
+  });
+}
+
+// GET /api/media/stats — registered before /api/media/:id so it isn't an id.
+app.get('/api/media/stats', async (c) => {
+  try {
+    return c.json({ ok: true, ...(await media.stats()) }, 200, { 'Cache-Control': 'no-store' });
+  } catch (err) {
+    return c.json({ ok: false, error: err.message }, 500);
+  }
+});
+
+// GET /api/media/:id?g=<gen>[&a=1] — the cached mp4 (or its m4a sidecar).
+// A URL with ?g= is immutable: a replaced copy gets a new gen, and an old gen
+// is served only while its file still exists (never different bytes).
+app.get('/api/media/:id', async (c) => {
+  const id = c.req.param('id');
+  const g = c.req.query('g');
+  const audio = c.req.query('a') === '1';
+  const path = await media.filePath(id, g, audio ? 'm4a' : 'mp4');
+  if (!path) return c.json({ ok: false, error: 'not cached' }, 404);
+  media.touch(id);
+  return rangeFileResponse(c, path, audio ? 'audio/mp4' : 'video/mp4',
+    g ? 'public, max-age=31536000, immutable' : 'no-cache');
+});
+
+// GET /api/media/:id/status
+app.get('/api/media/:id/status', async (c) => {
+  try {
+    return c.json({ ok: true, ...(await media.status(c.req.param('id'))) }, 200, { 'Cache-Control': 'no-store' });
+  } catch (err) {
+    return c.json({ ok: false, error: err.message }, 500);
+  }
+});
+
+// POST /api/media/:id/redownload — the client's "Broken" button. Idempotent:
+// a job already queued/running for this id is just reported back.
+app.post('/api/media/:id/redownload', async (c) => {
+  try {
+    return c.json({ ok: true, ...(await media.redownload(c.req.param('id'))) });
+  } catch (err) {
+    return c.json({ ok: false, error: err.message }, 400);
+  }
+});
+
+// Stream the server-cached copy as a save (OPFS writes it on the device).
+function cachedDownloadResponse(videoId, fp, row) {
+  const file = Bun.file(`${MEDIA_DIR}/${videoId}.${row.gen}.mp4`);
+  if (fp) recordVideoAccess(fp, { id: videoId }).catch(() => {});
+  media.touch(videoId);
+  return new Response(file, {
+    status: 200,
+    headers: {
+      'Content-Type':        'video/mp4',
+      'Content-Length':      String(file.size),
+      'Content-Disposition': `attachment; filename="${videoId}.mp4"`,
+      'Cache-Control':       'no-store',
+      'Access-Control-Allow-Origin': '*',
+    },
+  });
+}
+
 // GET /api/download/:videoId
-// Downloads the video server-side via yt-dlp and streams the finished file
-// to the browser so OPFS can store it. The browser never contacts YouTube
-// CDN directly (CORS would block it).
+// Saves go through the server cache: the fetch runs as a server-owned job
+// (a disconnecting client no longer kills it — the copy lands for next
+// time), then the validated file is streamed so OPFS can store it. The
+// browser never contacts YouTube CDN directly (CORS would block it).
 app.get('/api/download/:videoId', async (c) => {
   const videoId = (c.req.param('videoId') || '').replace(/[/\\:?<>|*"]/g, '').trim();
   if (!videoId) return c.json({ ok: false, error: 'missing videoId' }, 400);
@@ -917,17 +1104,33 @@ app.get('/api/download/:videoId', async (c) => {
     const keep = parseKeepParam(c.req.query('keep') || '');
     if (!keep.length) return c.json({ ok: false, error: 'missing or invalid keep segments' }, 400);
     try {
-      return await ytdlpEditedDownloadResponse(videoId, fp, keep, c.req.raw.signal);
+      const row = await media.ensureCached(videoId, { priority: HIGH }).catch(() => null);
+      const src = row ? await media.filePath(videoId, row.gen) : null;
+      return await ytdlpEditedDownloadResponse(videoId, fp, keep, c.req.raw.signal, src);
     } catch (err) {
       return c.json({ ok: false, error: err.message }, 500);
     }
   }
 
+  // Server cache first — plain saves and ?mux=1 both resolve to the same
+  // canonical ≤720p H.264 + AAC copy.
+  let cacheErr = null;
+  try {
+    const row = await media.ensureCached(videoId, { priority: HIGH });
+    return cachedDownloadResponse(videoId, fp, row);
+  } catch (err) {
+    cacheErr = err;
+    console.warn(`[ytplayer] cache save unavailable for ${videoId} (${err.code || 'error'}): ${err.message}`);
+  }
+
+  // Legacy per-request saves — only when the cache can't hold this video.
+  // A cache job that downloaded and FAILED validation already tried the mux
+  // format, so go straight to the progressive-first default in that case.
   // ?mux=1 — "Save before playing" path: bestvideo up to 720p PLUS bestaudio
   // compiled into one mp4 with ffmpeg on the server. Falls back to the
   // progressive single-file save below when ffmpeg is missing or the merge
   // fails.
-  if (c.req.query('mux') === '1') {
+  if (c.req.query('mux') === '1' && cacheErr && cacheErr.code === 'SKIPPED') {
     try {
       return await ytdlpDownloadResponse(videoId, fp, [
         '-f', SAVE_MUX_FORMAT,
@@ -1388,6 +1591,7 @@ app.get('/*', serveStatic({ path: './public/index.html' }));
 async function main() {
   await initDb();
   console.log(`[ytplayer] DB ready`);
+  await media.init();
   console.log(`[ytplayer] Starting on port ${PORT}`);
 
   // Bun.serve is the native Bun HTTP server