diff --git a/docker-compose.yml b/docker-compose.yml index 954b24c..6089f9b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,10 +5,11 @@ services: volumes: # libsql DB file persists across container rebuilds - ytplayer-data:/app/data - # Bulk, re-fetchable media on the homelab's USB drive (/mnt/data, mounted - # nofail). Only the media cache lives here — the DB, lyrics, notes and - # uploads stay on ytplayer-data. If the drive is missing, the marker file - # below is missing too and the cache pauses: playback streams instead. + # Bulk media on the homelab's USB drive (/mnt/data, mounted nofail): the + # re-fetchable media cache and the uploads library. The DB, lyrics and + # notes stay on ytplayer-data. If the drive is missing, the marker file + # below is missing too: the cache pauses (playback streams) and uploads + # are served from the nightly backup copy on ytplayer-data. - /mnt/data/ytplayer:/app/bulk environment: PORT: "3000" @@ -16,6 +17,11 @@ services: MEDIA_VOLUME_MARKER: "/app/bulk/.hl-data" MEDIA_CACHE_MAX_BYTES: "268435456000" # 250 GiB of the 458 GiB drive MEDIA_MIN_FREE_BYTES: "21474836480" # keep 20 GiB free on the drive + UPLOAD_DIR: "/app/bulk/uploads" + UPLOAD_VOLUME_MARKER: "/app/bulk/.hl-data" + # Second copy on the stable disk, refreshed nightly by + # scripts/ops/uploads-backup.sh on the homelab host; reads fall back to it. + UPLOAD_BACKUP_DIR: "/app/data/uploads-backup" DB_PATH: "/app/data/ytplayer.db" APP_VERSION: "1.0.0" # Unlocks /admin (API tokens, lyric/chapter history + restore). Set it in diff --git a/frontend/app.js b/frontend/app.js index 82d70ae..89f2c8b 100755 --- a/frontend/app.js +++ b/frontend/app.js @@ -7519,7 +7519,9 @@ async function openChannel(channelKey, displayName) { // The identifier we hand the backend to look a channel up (URL preferred). function channelKeyOf(v) { - return (v && (v.channelUrl || v.channelId)) || ''; + // Older saved entries may only have the channel's name; the server resolves + // a name (e.g. "Petra - Topic") to its channel id. + return (v && (v.channelUrl || v.channelId || v.channel)) || ''; } // ============================================================================ diff --git a/scripts/ops/uploads-backup.sh b/scripts/ops/uploads-backup.sh new file mode 100755 index 0000000..b9c4892 --- /dev/null +++ b/scripts/ops/uploads-backup.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# Nightly second copy of the ytplayer uploads library (homelab host, cron). +# +# USB drive (primary): /mnt/data/ytplayer/uploads +# stable disk (backup): /uploads-backup +# +# Copies new files both ways and never deletes: the server removes a deleted +# upload from both places itself, and anything uploaded while the drive was +# offline (written to the backup) is copied back onto the drive here. +set -eu +VOL=${VOL:-/var/lib/docker/volumes/ytplayer-main-1dihzn_ytplayer-data/_data} +USB=/mnt/data/ytplayer +MIN_FREE_KB=${MIN_FREE_KB:-20971520} # keep 20 GiB free on the stable disk +log() { logger -t ytplayer-uploads-backup "$*"; echo "$*"; } + +[ -f "$USB/.hl-data" ] || { log "USB drive offline (no marker) — skipped"; exit 0; } +mkdir -p "$USB/uploads" "$VOL/uploads-backup" + +need=$(du -sk "$USB/uploads" | cut -f1) +have=$(du -sk "$VOL/uploads-backup" | cut -f1) +free=$(df -Pk "$VOL" | awk 'NR==2{print $4}') +if [ $((free - (need - have))) -lt "$MIN_FREE_KB" ]; then + log "not enough room on the stable disk for the backup (free ${free} KB, need ~$((need - have)) KB more) — skipped" + exit 1 +fi + +rsync -a --update "$USB/uploads/" "$VOL/uploads-backup/" # newer wins (e.g. replaced cover art) +rsync -a --update "$VOL/uploads-backup/" "$USB/uploads/" +log "ok — $(ls "$USB/uploads" | wc -l) files, $(du -sh "$USB/uploads" | cut -f1)" diff --git a/server/server.js b/server/server.js index 71d9d17..9140b26 100644 --- a/server/server.js +++ b/server/server.js @@ -371,6 +371,22 @@ function channelUrlCandidates(c) { return out; } +// Name → channel id: the first search hit published by a channel with exactly +// that name (case-insensitive). " - Topic" names come from YouTube Music. +async function channelIdForName(name) { + const want = name.trim().toLowerCase(); + const out = await runYtdlpResilient([`ytsearch10:${name}`, '--dump-json', '--flat-playlist', '--no-warnings', '--ignore-errors'], { pooled: true }); + for (const line of out.split('\n')) { + try { + const j = JSON.parse(line); + const ch = String(j.channel || j.uploader || '').trim().toLowerCase(); + const id = j.channel_id || j.uploader_id; + if (ch === want && /^UC[\w-]{22}$/.test(id || '')) return id; + } catch { /* not JSON */ } + } + return null; +} + // Resolve a channel identifier to a /videos URL yt-dlp can fetch function channelToUrl(c) { let base = c.trim(); @@ -517,6 +533,21 @@ app.get('/api/channel', async (c) => { if (results.length) break; } catch (err) { lastErr = err; } } + // A bare name ("Maverick City Music - Topic") is not a handle: find the + // channel id through a search, then try its uploads/home again. + if (!results.length && !/^(https?:|@|UC[\w-]{22}$)/.test(chan)) { + const id = await channelIdForName(chan).catch(() => null); + if (id) { + for (const url of channelUrlCandidates(id)) { + try { + out = await runYtdlpResilient([url, '--dump-json', '--flat-playlist', '--no-warnings', '--ignore-errors', + '--playlist-end', String(CHANNEL_LIMIT)], { pooled: true }); + results = parseCards(out); + if (results.length) { lastErr = null; break; } + } catch (err) { lastErr = err; } + } + } + } if (!results.length && lastErr) throw lastErr; // Extract channel name + URL from the first record const first = results[0]; @@ -2050,6 +2081,8 @@ const uploads = registerUploadRoutes(app, { ffmpeg: FFMPEG, ffprobe: FFPROBE, uploadDir: UPLOAD_DIR, + backupDir: process.env.UPLOAD_BACKUP_DIR || null, + volumeMarker: process.env.UPLOAD_VOLUME_MARKER || null, rangeFileResponse, requireAdminOrToken: notes.requireAdminOrToken, parseLrc, diff --git a/server/uploads.js b/server/uploads.js index 80a3d02..c167f5c 100644 --- a/server/uploads.js +++ b/server/uploads.js @@ -82,10 +82,28 @@ export function describeProbe(probe, fallbackName) { export function registerUploadRoutes(app, deps) { const { db, ffmpeg = 'ffmpeg', ffprobe = 'ffprobe', uploadDir, rangeFileResponse, requireAdminOrToken, parseLrc, sanitizeLyrics } = deps; - mkdirSync(uploadDir, { recursive: true }); - - const filePath = (u) => join(uploadDir, `${u.id}.${u.ext}`); - const artPath = (id) => join(uploadDir, `${id}.art.jpg`); + // uploadDir may sit on a removable drive (volumeMarker = a file that only + // exists on it). backupDir is the nightly second copy on the stable disk: + // reads fall back to it, and while the drive is offline new uploads land + // there (the nightly sync copies them back over). + const { backupDir = null, volumeMarker = null } = deps; + const online = () => { if (!volumeMarker) return true; try { return existsSync(volumeMarker); } catch { return false; } }; + const writeDir = () => { + const d = online() || !backupDir ? uploadDir : backupDir; + try { mkdirSync(d, { recursive: true }); } catch { /* reported by the write itself */ } + return d; + }; + const dirs = () => [online() ? uploadDir : null, backupDir].filter(Boolean); + if (online()) mkdirSync(uploadDir, { recursive: true }); + if (backupDir) mkdirSync(backupDir, { recursive: true }); + const locate = (name) => { + for (const d of dirs()) { const p = join(d, name); if (existsSync(p)) return p; } + return join(writeDir(), name); + }; + const filePath = (u) => locate(`${u.id}.${u.ext}`); + const artPath = (id) => locate(`${id}.art.jpg`); + const newArtPath = (id) => join(writeDir(), `${id}.art.jpg`); + const removeEverywhere = (name) => { for (const d of [uploadDir, backupDir].filter(Boolean)) { try { unlinkSync(join(d, name)); } catch { /* gone */ } } }; async function probeFile(path) { const out = await run(ffprobe, ['-v', 'error', '-print_format', 'json', '-show_format', '-show_streams', path]); @@ -102,17 +120,17 @@ export function registerUploadRoutes(app, deps) { // Cover art: an uploaded image wins, else the embedded picture. let art = null; if (artFile && typeof artFile !== 'string' && artFile.size) { - const tmp = join(uploadDir, `${id}.art.src`); + const tmp = join(writeDir(), `${id}.art.src`); await Bun.write(tmp, artFile); try { - await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", artPath(id)]); + await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", newArtPath(id)]); art = 'file'; } catch { /* unusable image — ignore */ } try { unlinkSync(tmp); } catch { /* gone */ } } if (!art && info.coverIndex !== null) { try { - await run(ffmpeg, ['-v', 'error', '-y', '-i', target, '-map', `0:${info.coverIndex}`, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", artPath(id)]); + await run(ffmpeg, ['-v', 'error', '-y', '-i', target, '-map', `0:${info.coverIndex}`, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", newArtPath(id)]); art = 'embedded'; } catch { /* cover we can't decode */ } } @@ -144,7 +162,7 @@ export function registerUploadRoutes(app, deps) { const extError = (ext) => `unsupported file type ".${ext}" — video: ${[...VIDEO_EXT].join(', ')}; audio: ${[...AUDIO_EXT].join(', ')}`; const cleanup = (id, target) => { try { unlinkSync(target); } catch { /* not written */ } - try { unlinkSync(artPath(id)); } catch { /* none */ } + removeEverywhere(`${id}.art.jpg`); }; // Multipart upload — fine for small files and scripts, but the whole body is @@ -158,7 +176,7 @@ export function registerUploadRoutes(app, deps) { const ext = extOf(file.name); if (!VIDEO_EXT.has(ext) && !AUDIO_EXT.has(ext)) return c.json({ ok: false, error: extError(ext) }, 415); const id = 'upl_' + randomBytes(6).toString('hex'); - const target = join(uploadDir, `${id}.${ext}`); + const target = join(writeDir(), `${id}.${ext}`); try { await Bun.write(target, file); const r = await ingest({ id, ext, target, size: file.size, filename: file.name, fields: body, artFile: body.art }); @@ -181,7 +199,7 @@ export function registerUploadRoutes(app, deps) { const reader = c.req.raw.body ? c.req.raw.body.getReader() : null; if (!reader) return c.json({ ok: false, error: 'empty body' }, 400); const id = 'upl_' + randomBytes(6).toString('hex'); - const target = join(uploadDir, `${id}.${ext}`); + const target = join(writeDir(), `${id}.${ext}`); try { let got = 0; const out = createWriteStream(target); @@ -213,10 +231,10 @@ export function registerUploadRoutes(app, deps) { if (!UPLOAD_ID_RE.test(id) || !(await db.getUpload(id))) return c.json({ ok: false, error: 'not found' }, 404); const buf = await c.req.arrayBuffer(); if (!buf.byteLength || buf.byteLength > 20 * 1024 * 1024) return c.json({ ok: false, error: 'send an image under 20 MB' }, 400); - const tmp = join(uploadDir, `${id}.art.src`); + const tmp = join(writeDir(), `${id}.art.src`); try { await Bun.write(tmp, buf); - await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", artPath(id)]); + await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", newArtPath(id)]); await db.setUploadArt(id, 'file'); return c.json({ ok: true }); } catch (err) { @@ -232,8 +250,8 @@ export function registerUploadRoutes(app, deps) { const u = await db.getUpload(id); if (!u) return c.json({ ok: false, error: 'not found' }, 404); await db.deleteUpload(id); - try { unlinkSync(filePath(u)); } catch { /* gone */ } - try { unlinkSync(artPath(id)); } catch { /* none */ } + removeEverywhere(`${u.id}.${u.ext}`); + removeEverywhere(`${id}.art.jpg`); return c.json({ ok: true }); }); diff --git a/server/uploads.test.js b/server/uploads.test.js index 4e9ef7b..9409eac 100644 --- a/server/uploads.test.js +++ b/server/uploads.test.js @@ -165,3 +165,30 @@ describe('streaming upload (large / long files)', () => { expect((await db.getUpload(up.upload.id)).art).toBeTruthy(); }); }); + +describe('upload storage on a removable drive', () => { + test('reads the drive copy, falls back to the backup copy, and writes to the backup while offline', async () => { + const { mkdtempSync, writeFileSync, rmSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const { registerUploadRoutes } = await import('./uploads.js'); + const root = mkdtempSync(join(tmpdir(), 'ytp-up-')); + const drive = join(root, 'usb'), uploadDir = join(drive, 'uploads'), backupDir = join(root, 'backup'); + const marker = join(drive, '.hl-data'); + (await import('node:fs')).mkdirSync(drive, { recursive: true }); + writeFileSync(marker, ''); + const noop = () => {}; + const app = new Proxy({}, { get: () => noop }); + const up = registerUploadRoutes(app, { db: {}, uploadDir, backupDir, volumeMarker: marker, rangeFileResponse: noop, requireAdminOrToken: noop }); + const u = { id: 'up_test1', ext: 'mp4' }; + + writeFileSync(join(backupDir, 'up_test1.mp4'), 'backup'); + expect(up.filePath(u)).toBe(join(backupDir, 'up_test1.mp4')); // drive copy missing → backup + writeFileSync(join(uploadDir, 'up_test1.mp4'), 'drive'); + expect(up.filePath(u)).toBe(join(uploadDir, 'up_test1.mp4')); // drive copy preferred + rmSync(marker); + expect(up.filePath(u)).toBe(join(backupDir, 'up_test1.mp4')); // drive offline → backup + expect(up.filePath({ id: 'up_new', ext: 'mp4' })).toBe(join(backupDir, 'up_new.mp4')); // new writes go to the stable disk + rmSync(root, { recursive: true, force: true }); + }); +});