Uploads library on the USB drive with a nightly stable-disk copy; Topic channels resolve by name
- UPLOAD_DIR moves to the USB drive; reads fall back to UPLOAD_BACKUP_DIR on the
ytplayer-data volume, and while the drive is offline new uploads land there
- scripts/ops/uploads-backup.sh (homelab cron, 03:30) copies both ways, never
deletes; the server removes a deleted upload from both places
- /api/channel resolves a bare channel name ('Artist - Topic') to its id via a
search, and the client falls back to the channel name for old saved entries
This commit is contained in:
Jonathan Sykes
@@ -5,10 +5,11 @@ services:
|
||||
volumes:
|
||||
# libsql DB file persists across container rebuilds
|
||||
- ytplayer-data:/app/data
|
||||
# Bulk, re-fetchable media on the homelab's USB drive (/mnt/data, mounted
|
||||
# nofail). Only the media cache lives here — the DB, lyrics, notes and
|
||||
# uploads stay on ytplayer-data. If the drive is missing, the marker file
|
||||
# below is missing too and the cache pauses: playback streams instead.
|
||||
# Bulk media on the homelab's USB drive (/mnt/data, mounted nofail): the
|
||||
# re-fetchable media cache and the uploads library. The DB, lyrics and
|
||||
# notes stay on ytplayer-data. If the drive is missing, the marker file
|
||||
# below is missing too: the cache pauses (playback streams) and uploads
|
||||
# are served from the nightly backup copy on ytplayer-data.
|
||||
- /mnt/data/ytplayer:/app/bulk
|
||||
environment:
|
||||
PORT: "3000"
|
||||
@@ -16,6 +17,11 @@ services:
|
||||
MEDIA_VOLUME_MARKER: "/app/bulk/.hl-data"
|
||||
MEDIA_CACHE_MAX_BYTES: "268435456000" # 250 GiB of the 458 GiB drive
|
||||
MEDIA_MIN_FREE_BYTES: "21474836480" # keep 20 GiB free on the drive
|
||||
UPLOAD_DIR: "/app/bulk/uploads"
|
||||
UPLOAD_VOLUME_MARKER: "/app/bulk/.hl-data"
|
||||
# Second copy on the stable disk, refreshed nightly by
|
||||
# scripts/ops/uploads-backup.sh on the homelab host; reads fall back to it.
|
||||
UPLOAD_BACKUP_DIR: "/app/data/uploads-backup"
|
||||
DB_PATH: "/app/data/ytplayer.db"
|
||||
APP_VERSION: "1.0.0"
|
||||
# Unlocks /admin (API tokens, lyric/chapter history + restore). Set it in
|
||||
|
||||
@@ -7519,7 +7519,9 @@ async function openChannel(channelKey, displayName) {
|
||||
|
||||
// The identifier we hand the backend to look a channel up (URL preferred).
|
||||
function channelKeyOf(v) {
|
||||
return (v && (v.channelUrl || v.channelId)) || '';
|
||||
// Older saved entries may only have the channel's name; the server resolves
|
||||
// a name (e.g. "Petra - Topic") to its channel id.
|
||||
return (v && (v.channelUrl || v.channelId || v.channel)) || '';
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
|
||||
29
scripts/ops/uploads-backup.sh
Executable file
29
scripts/ops/uploads-backup.sh
Executable file
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
# Nightly second copy of the ytplayer uploads library (homelab host, cron).
|
||||
#
|
||||
# USB drive (primary): /mnt/data/ytplayer/uploads
|
||||
# stable disk (backup): <ytplayer-data volume>/uploads-backup
|
||||
#
|
||||
# Copies new files both ways and never deletes: the server removes a deleted
|
||||
# upload from both places itself, and anything uploaded while the drive was
|
||||
# offline (written to the backup) is copied back onto the drive here.
|
||||
set -eu
|
||||
VOL=${VOL:-/var/lib/docker/volumes/ytplayer-main-1dihzn_ytplayer-data/_data}
|
||||
USB=/mnt/data/ytplayer
|
||||
MIN_FREE_KB=${MIN_FREE_KB:-20971520} # keep 20 GiB free on the stable disk
|
||||
log() { logger -t ytplayer-uploads-backup "$*"; echo "$*"; }
|
||||
|
||||
[ -f "$USB/.hl-data" ] || { log "USB drive offline (no marker) — skipped"; exit 0; }
|
||||
mkdir -p "$USB/uploads" "$VOL/uploads-backup"
|
||||
|
||||
need=$(du -sk "$USB/uploads" | cut -f1)
|
||||
have=$(du -sk "$VOL/uploads-backup" | cut -f1)
|
||||
free=$(df -Pk "$VOL" | awk 'NR==2{print $4}')
|
||||
if [ $((free - (need - have))) -lt "$MIN_FREE_KB" ]; then
|
||||
log "not enough room on the stable disk for the backup (free ${free} KB, need ~$((need - have)) KB more) — skipped"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rsync -a --update "$USB/uploads/" "$VOL/uploads-backup/" # newer wins (e.g. replaced cover art)
|
||||
rsync -a --update "$VOL/uploads-backup/" "$USB/uploads/"
|
||||
log "ok — $(ls "$USB/uploads" | wc -l) files, $(du -sh "$USB/uploads" | cut -f1)"
|
||||
@@ -371,6 +371,22 @@ function channelUrlCandidates(c) {
|
||||
return out;
|
||||
}
|
||||
|
||||
// Name → channel id: the first search hit published by a channel with exactly
|
||||
// that name (case-insensitive). "<Artist> - Topic" names come from YouTube Music.
|
||||
async function channelIdForName(name) {
|
||||
const want = name.trim().toLowerCase();
|
||||
const out = await runYtdlpResilient([`ytsearch10:${name}`, '--dump-json', '--flat-playlist', '--no-warnings', '--ignore-errors'], { pooled: true });
|
||||
for (const line of out.split('\n')) {
|
||||
try {
|
||||
const j = JSON.parse(line);
|
||||
const ch = String(j.channel || j.uploader || '').trim().toLowerCase();
|
||||
const id = j.channel_id || j.uploader_id;
|
||||
if (ch === want && /^UC[\w-]{22}$/.test(id || '')) return id;
|
||||
} catch { /* not JSON */ }
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Resolve a channel identifier to a /videos URL yt-dlp can fetch
|
||||
function channelToUrl(c) {
|
||||
let base = c.trim();
|
||||
@@ -517,6 +533,21 @@ app.get('/api/channel', async (c) => {
|
||||
if (results.length) break;
|
||||
} catch (err) { lastErr = err; }
|
||||
}
|
||||
// A bare name ("Maverick City Music - Topic") is not a handle: find the
|
||||
// channel id through a search, then try its uploads/home again.
|
||||
if (!results.length && !/^(https?:|@|UC[\w-]{22}$)/.test(chan)) {
|
||||
const id = await channelIdForName(chan).catch(() => null);
|
||||
if (id) {
|
||||
for (const url of channelUrlCandidates(id)) {
|
||||
try {
|
||||
out = await runYtdlpResilient([url, '--dump-json', '--flat-playlist', '--no-warnings', '--ignore-errors',
|
||||
'--playlist-end', String(CHANNEL_LIMIT)], { pooled: true });
|
||||
results = parseCards(out);
|
||||
if (results.length) { lastErr = null; break; }
|
||||
} catch (err) { lastErr = err; }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!results.length && lastErr) throw lastErr;
|
||||
// Extract channel name + URL from the first record
|
||||
const first = results[0];
|
||||
@@ -2050,6 +2081,8 @@ const uploads = registerUploadRoutes(app, {
|
||||
ffmpeg: FFMPEG,
|
||||
ffprobe: FFPROBE,
|
||||
uploadDir: UPLOAD_DIR,
|
||||
backupDir: process.env.UPLOAD_BACKUP_DIR || null,
|
||||
volumeMarker: process.env.UPLOAD_VOLUME_MARKER || null,
|
||||
rangeFileResponse,
|
||||
requireAdminOrToken: notes.requireAdminOrToken,
|
||||
parseLrc,
|
||||
|
||||
@@ -82,10 +82,28 @@ export function describeProbe(probe, fallbackName) {
|
||||
|
||||
export function registerUploadRoutes(app, deps) {
|
||||
const { db, ffmpeg = 'ffmpeg', ffprobe = 'ffprobe', uploadDir, rangeFileResponse, requireAdminOrToken, parseLrc, sanitizeLyrics } = deps;
|
||||
mkdirSync(uploadDir, { recursive: true });
|
||||
|
||||
const filePath = (u) => join(uploadDir, `${u.id}.${u.ext}`);
|
||||
const artPath = (id) => join(uploadDir, `${id}.art.jpg`);
|
||||
// uploadDir may sit on a removable drive (volumeMarker = a file that only
|
||||
// exists on it). backupDir is the nightly second copy on the stable disk:
|
||||
// reads fall back to it, and while the drive is offline new uploads land
|
||||
// there (the nightly sync copies them back over).
|
||||
const { backupDir = null, volumeMarker = null } = deps;
|
||||
const online = () => { if (!volumeMarker) return true; try { return existsSync(volumeMarker); } catch { return false; } };
|
||||
const writeDir = () => {
|
||||
const d = online() || !backupDir ? uploadDir : backupDir;
|
||||
try { mkdirSync(d, { recursive: true }); } catch { /* reported by the write itself */ }
|
||||
return d;
|
||||
};
|
||||
const dirs = () => [online() ? uploadDir : null, backupDir].filter(Boolean);
|
||||
if (online()) mkdirSync(uploadDir, { recursive: true });
|
||||
if (backupDir) mkdirSync(backupDir, { recursive: true });
|
||||
const locate = (name) => {
|
||||
for (const d of dirs()) { const p = join(d, name); if (existsSync(p)) return p; }
|
||||
return join(writeDir(), name);
|
||||
};
|
||||
const filePath = (u) => locate(`${u.id}.${u.ext}`);
|
||||
const artPath = (id) => locate(`${id}.art.jpg`);
|
||||
const newArtPath = (id) => join(writeDir(), `${id}.art.jpg`);
|
||||
const removeEverywhere = (name) => { for (const d of [uploadDir, backupDir].filter(Boolean)) { try { unlinkSync(join(d, name)); } catch { /* gone */ } } };
|
||||
|
||||
async function probeFile(path) {
|
||||
const out = await run(ffprobe, ['-v', 'error', '-print_format', 'json', '-show_format', '-show_streams', path]);
|
||||
@@ -102,17 +120,17 @@ export function registerUploadRoutes(app, deps) {
|
||||
// Cover art: an uploaded image wins, else the embedded picture.
|
||||
let art = null;
|
||||
if (artFile && typeof artFile !== 'string' && artFile.size) {
|
||||
const tmp = join(uploadDir, `${id}.art.src`);
|
||||
const tmp = join(writeDir(), `${id}.art.src`);
|
||||
await Bun.write(tmp, artFile);
|
||||
try {
|
||||
await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", artPath(id)]);
|
||||
await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", newArtPath(id)]);
|
||||
art = 'file';
|
||||
} catch { /* unusable image — ignore */ }
|
||||
try { unlinkSync(tmp); } catch { /* gone */ }
|
||||
}
|
||||
if (!art && info.coverIndex !== null) {
|
||||
try {
|
||||
await run(ffmpeg, ['-v', 'error', '-y', '-i', target, '-map', `0:${info.coverIndex}`, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", artPath(id)]);
|
||||
await run(ffmpeg, ['-v', 'error', '-y', '-i', target, '-map', `0:${info.coverIndex}`, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", newArtPath(id)]);
|
||||
art = 'embedded';
|
||||
} catch { /* cover we can't decode */ }
|
||||
}
|
||||
@@ -144,7 +162,7 @@ export function registerUploadRoutes(app, deps) {
|
||||
const extError = (ext) => `unsupported file type ".${ext}" — video: ${[...VIDEO_EXT].join(', ')}; audio: ${[...AUDIO_EXT].join(', ')}`;
|
||||
const cleanup = (id, target) => {
|
||||
try { unlinkSync(target); } catch { /* not written */ }
|
||||
try { unlinkSync(artPath(id)); } catch { /* none */ }
|
||||
removeEverywhere(`${id}.art.jpg`);
|
||||
};
|
||||
|
||||
// Multipart upload — fine for small files and scripts, but the whole body is
|
||||
@@ -158,7 +176,7 @@ export function registerUploadRoutes(app, deps) {
|
||||
const ext = extOf(file.name);
|
||||
if (!VIDEO_EXT.has(ext) && !AUDIO_EXT.has(ext)) return c.json({ ok: false, error: extError(ext) }, 415);
|
||||
const id = 'upl_' + randomBytes(6).toString('hex');
|
||||
const target = join(uploadDir, `${id}.${ext}`);
|
||||
const target = join(writeDir(), `${id}.${ext}`);
|
||||
try {
|
||||
await Bun.write(target, file);
|
||||
const r = await ingest({ id, ext, target, size: file.size, filename: file.name, fields: body, artFile: body.art });
|
||||
@@ -181,7 +199,7 @@ export function registerUploadRoutes(app, deps) {
|
||||
const reader = c.req.raw.body ? c.req.raw.body.getReader() : null;
|
||||
if (!reader) return c.json({ ok: false, error: 'empty body' }, 400);
|
||||
const id = 'upl_' + randomBytes(6).toString('hex');
|
||||
const target = join(uploadDir, `${id}.${ext}`);
|
||||
const target = join(writeDir(), `${id}.${ext}`);
|
||||
try {
|
||||
let got = 0;
|
||||
const out = createWriteStream(target);
|
||||
@@ -213,10 +231,10 @@ export function registerUploadRoutes(app, deps) {
|
||||
if (!UPLOAD_ID_RE.test(id) || !(await db.getUpload(id))) return c.json({ ok: false, error: 'not found' }, 404);
|
||||
const buf = await c.req.arrayBuffer();
|
||||
if (!buf.byteLength || buf.byteLength > 20 * 1024 * 1024) return c.json({ ok: false, error: 'send an image under 20 MB' }, 400);
|
||||
const tmp = join(uploadDir, `${id}.art.src`);
|
||||
const tmp = join(writeDir(), `${id}.art.src`);
|
||||
try {
|
||||
await Bun.write(tmp, buf);
|
||||
await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", artPath(id)]);
|
||||
await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", newArtPath(id)]);
|
||||
await db.setUploadArt(id, 'file');
|
||||
return c.json({ ok: true });
|
||||
} catch (err) {
|
||||
@@ -232,8 +250,8 @@ export function registerUploadRoutes(app, deps) {
|
||||
const u = await db.getUpload(id);
|
||||
if (!u) return c.json({ ok: false, error: 'not found' }, 404);
|
||||
await db.deleteUpload(id);
|
||||
try { unlinkSync(filePath(u)); } catch { /* gone */ }
|
||||
try { unlinkSync(artPath(id)); } catch { /* none */ }
|
||||
removeEverywhere(`${u.id}.${u.ext}`);
|
||||
removeEverywhere(`${id}.art.jpg`);
|
||||
return c.json({ ok: true });
|
||||
});
|
||||
|
||||
|
||||
@@ -165,3 +165,30 @@ describe('streaming upload (large / long files)', () => {
|
||||
expect((await db.getUpload(up.upload.id)).art).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe('upload storage on a removable drive', () => {
|
||||
test('reads the drive copy, falls back to the backup copy, and writes to the backup while offline', async () => {
|
||||
const { mkdtempSync, writeFileSync, rmSync } = await import('node:fs');
|
||||
const { tmpdir } = await import('node:os');
|
||||
const { join } = await import('node:path');
|
||||
const { registerUploadRoutes } = await import('./uploads.js');
|
||||
const root = mkdtempSync(join(tmpdir(), 'ytp-up-'));
|
||||
const drive = join(root, 'usb'), uploadDir = join(drive, 'uploads'), backupDir = join(root, 'backup');
|
||||
const marker = join(drive, '.hl-data');
|
||||
(await import('node:fs')).mkdirSync(drive, { recursive: true });
|
||||
writeFileSync(marker, '');
|
||||
const noop = () => {};
|
||||
const app = new Proxy({}, { get: () => noop });
|
||||
const up = registerUploadRoutes(app, { db: {}, uploadDir, backupDir, volumeMarker: marker, rangeFileResponse: noop, requireAdminOrToken: noop });
|
||||
const u = { id: 'up_test1', ext: 'mp4' };
|
||||
|
||||
writeFileSync(join(backupDir, 'up_test1.mp4'), 'backup');
|
||||
expect(up.filePath(u)).toBe(join(backupDir, 'up_test1.mp4')); // drive copy missing → backup
|
||||
writeFileSync(join(uploadDir, 'up_test1.mp4'), 'drive');
|
||||
expect(up.filePath(u)).toBe(join(uploadDir, 'up_test1.mp4')); // drive copy preferred
|
||||
rmSync(marker);
|
||||
expect(up.filePath(u)).toBe(join(backupDir, 'up_test1.mp4')); // drive offline → backup
|
||||
expect(up.filePath({ id: 'up_new', ext: 'mp4' })).toBe(join(backupDir, 'up_new.mp4')); // new writes go to the stable disk
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user