From 1f9a97a6ae26ec711e68feb55b9b461f5b290cf4 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Fri, 2 Oct 2026 21:21:23 +0800 Subject: [PATCH] Media cache on the homelab USB drive with an offline guard; Topic channels list their tracks - MEDIA_DIR moves to /mnt/data/ytplayer (bind-mounted at /app/bulk); DB, lyrics, notes and uploads stay on the ytplayer-data volume - MEDIA_VOLUME_MARKER: while the marker on the drive is missing the cache writes, serves and deletes nothing, so playback streams and an unmounted drive never fills the disk under its mountpoint or loses its index on a reboot - Budget 250 GiB, keep 20 GiB free on the drive - /api/channel falls back to the uploads playlist and the channel home page when a channel has no Videos tab (auto-generated '- Topic' music channels) --- docker-compose.yml | 14 ++++++++++--- server/media-cache.js | 25 ++++++++++++++++++++--- server/media-cache.test.js | 40 ++++++++++++++++++++++++++++++++++++ server/server.js | 42 ++++++++++++++++++++++++++++++-------- 4 files changed, 106 insertions(+), 15 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0e096ff..954b24c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,8 +5,17 @@ services: volumes: # libsql DB file persists across container rebuilds - ytplayer-data:/app/data + # Bulk, re-fetchable media on the homelab's USB drive (/mnt/data, mounted + # nofail). Only the media cache lives here — the DB, lyrics, notes and + # uploads stay on ytplayer-data. If the drive is missing, the marker file + # below is missing too and the cache pauses: playback streams instead. + - /mnt/data/ytplayer:/app/bulk environment: PORT: "3000" + MEDIA_DIR: "/app/bulk/media" + MEDIA_VOLUME_MARKER: "/app/bulk/.hl-data" + MEDIA_CACHE_MAX_BYTES: "268435456000" # 250 GiB of the 458 GiB drive + MEDIA_MIN_FREE_BYTES: "21474836480" # keep 20 GiB free on the drive DB_PATH: "/app/data/ytplayer.db" APP_VERSION: "1.0.0" # Unlocks /admin (API tokens, lyric/chapter history + restore). Set it in @@ -36,9 +45,8 @@ services: # YTDLP_WORKER: "0" # disable the long-lived yt-dlp worker pool # YTDLP_WORKERS: "2" # pool size # Server media cache (server/media-cache.js) — defaults shown. - # MEDIA_DIR: "/app/data/media" # on the ytplayer-data volume - # MEDIA_CACHE_MAX_BYTES: "10737418240" # 10 GiB, LRU eviction past this - # MEDIA_MIN_FREE_BYTES: "5368709120" # skip caching below 5 GiB free disk + # (MEDIA_DIR / MEDIA_CACHE_MAX_BYTES / MEDIA_MIN_FREE_BYTES set above; + # defaults without them: /app/data/media, 10 GiB, 5 GiB) # MEDIA_AUTO_MAX_SECONDS: "3600" # longest video auto-cached on play # MEDIA_TRANSCODE: "1" # 0 disables the compression lane # MEDIA_CODEC: "hevc" # hevc | h264 diff --git a/server/media-cache.js b/server/media-cache.js index fc04a28..323522f 100644 --- a/server/media-cache.js +++ b/server/media-cache.js @@ -171,7 +171,17 @@ export function createMediaCache({ hashFile = sha256File, // server-computed SHA-256 = the file's P2P content id onReady = null, // ({ id, gen, path, sha256, size, … }) after a validated copy lands backfillDelayMs = 30_000, // hash pre-existing copies this long after init() + // Optional marker file that only exists on the real cache volume (e.g. a USB + // drive bind-mounted into the container). While it is missing the volume is + // treated as OFFLINE: nothing is written (so an unmounted drive never fills + // the disk underneath its mountpoint), nothing is served (playback streams + // instead), and nothing is deleted (a drive that comes back keeps its copies). + volumeMarker = null, } = {}) { + const available = () => { + if (!volumeMarker) return true; + try { return existsSync(volumeMarker); } catch { return false; } + }; const tmpDir = join(dir, '.tmp'); const fileFor = (id, gen, kind = 'mp4') => join(dir, `${id}.${gen}.${kind}`); @@ -208,7 +218,7 @@ export function createMediaCache({ } function readyFilesExist(row) { - return existsSync(fileFor(row.video_id, row.gen)) && existsSync(fileFor(row.video_id, row.gen, 'm4a')); + return available() && existsSync(fileFor(row.video_id, row.gen)) && existsSync(fileFor(row.video_id, row.gen, 'm4a')); } function safeFree() { @@ -223,6 +233,7 @@ export function createMediaCache({ // Make room for ~est bytes under the budget (LRU), then check the disk guard. async function makeRoom(id, est) { + if (!available()) throw new MediaSkip('cache volume offline — streaming instead'); let { bytes } = await db.mediaStats(); if (bytes + est > maxBytes) { const cutoff = now() - EVICT_PROTECT_MS; @@ -512,6 +523,7 @@ export function createMediaCache({ if (again) { bump(again, priority, auto); return again.promise; } if (row && row.status === 'ready') { if (readyFilesExist(row)) return row; + if (!available()) throw new MediaSkip('cache volume offline — streaming instead'); await db.deleteMedia(id); removeFiles(id); } else if (row && row.status === 'failed' && !force && now() < row.retry_at) { @@ -560,6 +572,7 @@ export function createMediaCache({ async function redownload(id) { if (!isMediaId(id)) throw new MediaSkip('invalid video id'); + if (!available()) throw new MediaSkip('cache volume offline — streaming instead'); const existing = jobs.get(id); if (existing) { bump(existing, HIGH, false); return { status: existing.status }; } const row = await db.getMedia(id); @@ -585,7 +598,7 @@ export function createMediaCache({ // Path for /api/media. With a gen, ONLY that generation is served (never // different bytes under the same URL); without one, the current copy. async function filePath(id, gen, kind = 'mp4') { - if (!isMediaId(id)) return null; + if (!isMediaId(id) || !available()) return null; let g = gen !== undefined && gen !== null && gen !== '' ? Number(gen) : null; if (g !== null && !Number.isInteger(g)) return null; if (g === null) { @@ -659,6 +672,12 @@ export function createMediaCache({ // Boot: sweep partial work, drop files/rows that disagree, resume jobs. async function init() { + if (!available()) { + // Reconciling now would drop every row (files "missing") and then, once + // the drive is back, delete every file as an orphan. Leave it all alone. + log.warn?.(`[media] cache volume offline (no ${volumeMarker}) — caching paused, playback streams`); + return; + } mkdirSync(tmpDir, { recursive: true }); sweepTmp(); const rows = await db.listMedia(); @@ -712,5 +731,5 @@ export function createMediaCache({ return n; } - return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile }; + return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available }; } diff --git a/server/media-cache.test.js b/server/media-cache.test.js index 05fa257..9e4e869 100644 --- a/server/media-cache.test.js +++ b/server/media-cache.test.js @@ -389,3 +389,43 @@ describe('media cache jobs', () => { expect(await cache.filePath('..', 1)).toBeNull(); }); }); + +describe('removable cache volume (volumeMarker)', () => { + test('offline volume: nothing is written, nothing served, rows survive a reboot', async () => { + await clearDb(); + const markerDir = mkdtempSync(join(tmpdir(), 'ytp-vol-')); + const marker = join(markerDir, '.hl-data'); + writeFileSync(marker, ''); + const seed = makeCache({ volumeMarker: marker }); + await seed.cache.init(); + const row = await seed.cache.ensureCached('volAAAAAAA1', { priority: HIGH }); + expect(row.status).toBe('ready'); + expect(await seed.cache.filePath('volAAAAAAA1')).not.toBeNull(); + + // Drive unplugged: the marker vanishes. + rmSync(marker); + expect(seed.cache.available()).toBe(false); + expect(await seed.cache.filePath('volAAAAAAA1')).toBeNull(); + expect(await seed.cache.getReady('volAAAAAAA1')).toBeNull(); + await expect(seed.cache.ensureCached('volAAAAAAA2', { priority: HIGH })).rejects.toBeInstanceOf(MediaSkip); + await expect(seed.cache.ensureCached('volAAAAAAA1', { priority: HIGH })).rejects.toBeInstanceOf(MediaSkip); + expect(seed.calls).toEqual(['volAAAAAAA1']); + + // Reboot while offline must not reconcile (it would drop every row). + const again = makeCache({ volumeMarker: marker, dir: seed.dir }); + await again.cache.init(); + expect((await dbmod.getMedia('volAAAAAAA1')).status).toBe('ready'); + expect(existsSync(join(seed.dir, `volAAAAAAA1.${row.gen}.mp4`))).toBe(true); + + // Drive back: the same copy is served again without a refetch. + writeFileSync(marker, ''); + expect(await again.cache.filePath('volAAAAAAA1')).not.toBeNull(); + rmSync(markerDir, { recursive: true, force: true }); + }); + + test('no marker configured behaves exactly as before', async () => { + await clearDb(); + const { cache } = makeCache(); + expect(cache.available()).toBe(true); + }); +}); diff --git a/server/server.js b/server/server.js index e2c6e24..71d9d17 100644 --- a/server/server.js +++ b/server/server.js @@ -357,6 +357,20 @@ function parseCards(output) { return results; } +// Auto-generated " - Topic" channels (YouTube Music) have no Videos +// tab, so /videos fails with "This channel does not have a videos tab". Their +// tracks are still in the channel's uploads playlist (UC… → UU…) and on the +// channel home page, so those are tried next. +function channelUrlCandidates(c) { + const videos = channelToUrl(c); + const root = videos.replace(/\/videos$/, ''); + const m = /(?:^|\/channel\/)(UC[\w-]{22})(?:[/?#]|$)/.exec(c.trim()) || /\/channel\/(UC[\w-]{22})/.exec(root); + const out = [videos]; + if (m) out.push(`https://www.youtube.com/playlist?list=UU${m[1].slice(2)}`); + out.push(root); + return out; +} + // Resolve a channel identifier to a /videos URL yt-dlp can fetch function channelToUrl(c) { let base = c.trim(); @@ -490,14 +504,20 @@ app.get('/api/channel', async (c) => { if (!chan) return c.json({ ok: false, error: 'missing channel' }, 400); try { - const url = channelToUrl(chan); - const out = await runYtdlpResilient([ - url, - '--dump-json', '--flat-playlist', - '--no-warnings', '--ignore-errors', - '--playlist-end', String(CHANNEL_LIMIT), - ], { pooled: true }); - const results = parseCards(out); + let out = '', results = [], lastErr = null; + for (const url of channelUrlCandidates(chan)) { + try { + out = await runYtdlpResilient([ + url, + '--dump-json', '--flat-playlist', + '--no-warnings', '--ignore-errors', + '--playlist-end', String(CHANNEL_LIMIT), + ], { pooled: true }); + results = parseCards(out); + if (results.length) break; + } catch (err) { lastErr = err; } + } + if (!results.length && lastErr) throw lastErr; // Extract channel name + URL from the first record const first = results[0]; let channelName = '', channelUrl = ''; @@ -1122,10 +1142,14 @@ async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal, cachedSrc // ============================================================================ const envNum = (k, d) => (process.env[k] !== undefined && process.env[k] !== '' ? Number(process.env[k]) : d); const MEDIA_DIR = process.env.MEDIA_DIR || './data/media'; -mkdirSync(MEDIA_DIR, { recursive: true }); +// Set when MEDIA_DIR sits on a removable drive: the cache only uses the +// directory while this marker (created once on the drive itself) exists. +const MEDIA_VOLUME_MARKER = process.env.MEDIA_VOLUME_MARKER || null; +if (!MEDIA_VOLUME_MARKER || existsSync(MEDIA_VOLUME_MARKER)) mkdirSync(MEDIA_DIR, { recursive: true }); const media = createMediaCache({ dir: MEDIA_DIR, + volumeMarker: MEDIA_VOLUME_MARKER, db: { getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats, // Retention (docs/p2p-architecture.md flow 9): cold copies go before popular ones.