diff --git a/docker-compose.yml b/docker-compose.yml index 0e096ff..954b24c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,8 +5,17 @@ services: volumes: # libsql DB file persists across container rebuilds - ytplayer-data:/app/data + # Bulk, re-fetchable media on the homelab's USB drive (/mnt/data, mounted + # nofail). Only the media cache lives here — the DB, lyrics, notes and + # uploads stay on ytplayer-data. If the drive is missing, the marker file + # below is missing too and the cache pauses: playback streams instead. + - /mnt/data/ytplayer:/app/bulk environment: PORT: "3000" + MEDIA_DIR: "/app/bulk/media" + MEDIA_VOLUME_MARKER: "/app/bulk/.hl-data" + MEDIA_CACHE_MAX_BYTES: "268435456000" # 250 GiB of the 458 GiB drive + MEDIA_MIN_FREE_BYTES: "21474836480" # keep 20 GiB free on the drive DB_PATH: "/app/data/ytplayer.db" APP_VERSION: "1.0.0" # Unlocks /admin (API tokens, lyric/chapter history + restore). Set it in @@ -36,9 +45,8 @@ services: # YTDLP_WORKER: "0" # disable the long-lived yt-dlp worker pool # YTDLP_WORKERS: "2" # pool size # Server media cache (server/media-cache.js) — defaults shown. - # MEDIA_DIR: "/app/data/media" # on the ytplayer-data volume - # MEDIA_CACHE_MAX_BYTES: "10737418240" # 10 GiB, LRU eviction past this - # MEDIA_MIN_FREE_BYTES: "5368709120" # skip caching below 5 GiB free disk + # (MEDIA_DIR / MEDIA_CACHE_MAX_BYTES / MEDIA_MIN_FREE_BYTES set above; + # defaults without them: /app/data/media, 10 GiB, 5 GiB) # MEDIA_AUTO_MAX_SECONDS: "3600" # longest video auto-cached on play # MEDIA_TRANSCODE: "1" # 0 disables the compression lane # MEDIA_CODEC: "hevc" # hevc | h264 diff --git a/server/media-cache.js b/server/media-cache.js index fc04a28..323522f 100644 --- a/server/media-cache.js +++ b/server/media-cache.js @@ -171,7 +171,17 @@ export function createMediaCache({ hashFile = sha256File, // server-computed SHA-256 = the file's P2P content id onReady = null, // ({ id, gen, path, sha256, size, … }) after a validated copy lands backfillDelayMs = 30_000, // hash pre-existing copies this long after init() + // Optional marker file that only exists on the real cache volume (e.g. a USB + // drive bind-mounted into the container). While it is missing the volume is + // treated as OFFLINE: nothing is written (so an unmounted drive never fills + // the disk underneath its mountpoint), nothing is served (playback streams + // instead), and nothing is deleted (a drive that comes back keeps its copies). + volumeMarker = null, } = {}) { + const available = () => { + if (!volumeMarker) return true; + try { return existsSync(volumeMarker); } catch { return false; } + }; const tmpDir = join(dir, '.tmp'); const fileFor = (id, gen, kind = 'mp4') => join(dir, `${id}.${gen}.${kind}`); @@ -208,7 +218,7 @@ export function createMediaCache({ } function readyFilesExist(row) { - return existsSync(fileFor(row.video_id, row.gen)) && existsSync(fileFor(row.video_id, row.gen, 'm4a')); + return available() && existsSync(fileFor(row.video_id, row.gen)) && existsSync(fileFor(row.video_id, row.gen, 'm4a')); } function safeFree() { @@ -223,6 +233,7 @@ export function createMediaCache({ // Make room for ~est bytes under the budget (LRU), then check the disk guard. async function makeRoom(id, est) { + if (!available()) throw new MediaSkip('cache volume offline — streaming instead'); let { bytes } = await db.mediaStats(); if (bytes + est > maxBytes) { const cutoff = now() - EVICT_PROTECT_MS; @@ -512,6 +523,7 @@ export function createMediaCache({ if (again) { bump(again, priority, auto); return again.promise; } if (row && row.status === 'ready') { if (readyFilesExist(row)) return row; + if (!available()) throw new MediaSkip('cache volume offline — streaming instead'); await db.deleteMedia(id); removeFiles(id); } else if (row && row.status === 'failed' && !force && now() < row.retry_at) { @@ -560,6 +572,7 @@ export function createMediaCache({ async function redownload(id) { if (!isMediaId(id)) throw new MediaSkip('invalid video id'); + if (!available()) throw new MediaSkip('cache volume offline — streaming instead'); const existing = jobs.get(id); if (existing) { bump(existing, HIGH, false); return { status: existing.status }; } const row = await db.getMedia(id); @@ -585,7 +598,7 @@ export function createMediaCache({ // Path for /api/media. With a gen, ONLY that generation is served (never // different bytes under the same URL); without one, the current copy. async function filePath(id, gen, kind = 'mp4') { - if (!isMediaId(id)) return null; + if (!isMediaId(id) || !available()) return null; let g = gen !== undefined && gen !== null && gen !== '' ? Number(gen) : null; if (g !== null && !Number.isInteger(g)) return null; if (g === null) { @@ -659,6 +672,12 @@ export function createMediaCache({ // Boot: sweep partial work, drop files/rows that disagree, resume jobs. async function init() { + if (!available()) { + // Reconciling now would drop every row (files "missing") and then, once + // the drive is back, delete every file as an orphan. Leave it all alone. + log.warn?.(`[media] cache volume offline (no ${volumeMarker}) — caching paused, playback streams`); + return; + } mkdirSync(tmpDir, { recursive: true }); sweepTmp(); const rows = await db.listMedia(); @@ -712,5 +731,5 @@ export function createMediaCache({ return n; } - return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile }; + return { init, ensureCached, redownload, verify, getReady, filePath, touch, status, stats, isMediaId, backfillHashes, adoptFile, available }; } diff --git a/server/media-cache.test.js b/server/media-cache.test.js index 05fa257..9e4e869 100644 --- a/server/media-cache.test.js +++ b/server/media-cache.test.js @@ -389,3 +389,43 @@ describe('media cache jobs', () => { expect(await cache.filePath('..', 1)).toBeNull(); }); }); + +describe('removable cache volume (volumeMarker)', () => { + test('offline volume: nothing is written, nothing served, rows survive a reboot', async () => { + await clearDb(); + const markerDir = mkdtempSync(join(tmpdir(), 'ytp-vol-')); + const marker = join(markerDir, '.hl-data'); + writeFileSync(marker, ''); + const seed = makeCache({ volumeMarker: marker }); + await seed.cache.init(); + const row = await seed.cache.ensureCached('volAAAAAAA1', { priority: HIGH }); + expect(row.status).toBe('ready'); + expect(await seed.cache.filePath('volAAAAAAA1')).not.toBeNull(); + + // Drive unplugged: the marker vanishes. + rmSync(marker); + expect(seed.cache.available()).toBe(false); + expect(await seed.cache.filePath('volAAAAAAA1')).toBeNull(); + expect(await seed.cache.getReady('volAAAAAAA1')).toBeNull(); + await expect(seed.cache.ensureCached('volAAAAAAA2', { priority: HIGH })).rejects.toBeInstanceOf(MediaSkip); + await expect(seed.cache.ensureCached('volAAAAAAA1', { priority: HIGH })).rejects.toBeInstanceOf(MediaSkip); + expect(seed.calls).toEqual(['volAAAAAAA1']); + + // Reboot while offline must not reconcile (it would drop every row). + const again = makeCache({ volumeMarker: marker, dir: seed.dir }); + await again.cache.init(); + expect((await dbmod.getMedia('volAAAAAAA1')).status).toBe('ready'); + expect(existsSync(join(seed.dir, `volAAAAAAA1.${row.gen}.mp4`))).toBe(true); + + // Drive back: the same copy is served again without a refetch. + writeFileSync(marker, ''); + expect(await again.cache.filePath('volAAAAAAA1')).not.toBeNull(); + rmSync(markerDir, { recursive: true, force: true }); + }); + + test('no marker configured behaves exactly as before', async () => { + await clearDb(); + const { cache } = makeCache(); + expect(cache.available()).toBe(true); + }); +}); diff --git a/server/server.js b/server/server.js index e2c6e24..71d9d17 100644 --- a/server/server.js +++ b/server/server.js @@ -357,6 +357,20 @@ function parseCards(output) { return results; } +// Auto-generated " - Topic" channels (YouTube Music) have no Videos +// tab, so /videos fails with "This channel does not have a videos tab". Their +// tracks are still in the channel's uploads playlist (UC… → UU…) and on the +// channel home page, so those are tried next. +function channelUrlCandidates(c) { + const videos = channelToUrl(c); + const root = videos.replace(/\/videos$/, ''); + const m = /(?:^|\/channel\/)(UC[\w-]{22})(?:[/?#]|$)/.exec(c.trim()) || /\/channel\/(UC[\w-]{22})/.exec(root); + const out = [videos]; + if (m) out.push(`https://www.youtube.com/playlist?list=UU${m[1].slice(2)}`); + out.push(root); + return out; +} + // Resolve a channel identifier to a /videos URL yt-dlp can fetch function channelToUrl(c) { let base = c.trim(); @@ -490,14 +504,20 @@ app.get('/api/channel', async (c) => { if (!chan) return c.json({ ok: false, error: 'missing channel' }, 400); try { - const url = channelToUrl(chan); - const out = await runYtdlpResilient([ - url, - '--dump-json', '--flat-playlist', - '--no-warnings', '--ignore-errors', - '--playlist-end', String(CHANNEL_LIMIT), - ], { pooled: true }); - const results = parseCards(out); + let out = '', results = [], lastErr = null; + for (const url of channelUrlCandidates(chan)) { + try { + out = await runYtdlpResilient([ + url, + '--dump-json', '--flat-playlist', + '--no-warnings', '--ignore-errors', + '--playlist-end', String(CHANNEL_LIMIT), + ], { pooled: true }); + results = parseCards(out); + if (results.length) break; + } catch (err) { lastErr = err; } + } + if (!results.length && lastErr) throw lastErr; // Extract channel name + URL from the first record const first = results[0]; let channelName = '', channelUrl = ''; @@ -1122,10 +1142,14 @@ async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal, cachedSrc // ============================================================================ const envNum = (k, d) => (process.env[k] !== undefined && process.env[k] !== '' ? Number(process.env[k]) : d); const MEDIA_DIR = process.env.MEDIA_DIR || './data/media'; -mkdirSync(MEDIA_DIR, { recursive: true }); +// Set when MEDIA_DIR sits on a removable drive: the cache only uses the +// directory while this marker (created once on the drive itself) exists. +const MEDIA_VOLUME_MARKER = process.env.MEDIA_VOLUME_MARKER || null; +if (!MEDIA_VOLUME_MARKER || existsSync(MEDIA_VOLUME_MARKER)) mkdirSync(MEDIA_DIR, { recursive: true }); const media = createMediaCache({ dir: MEDIA_DIR, + volumeMarker: MEDIA_VOLUME_MARKER, db: { getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats, // Retention (docs/p2p-architecture.md flow 9): cold copies go before popular ones.