josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Uploads library on the USB drive with a nightly stable-disk copy; Topic channels resolve by name

Browse Source 
- UPLOAD_DIR moves to the USB drive; reads fall back to UPLOAD_BACKUP_DIR on the
  ytplayer-data volume, and while the drive is offline new uploads land there
- scripts/ops/uploads-backup.sh (homelab cron, 03:30) copies both ways, never
  deletes; the server removes a deleted upload from both places
- /api/channel resolves a bare channel name ('Artist - Topic') to its id via a
  search, and the client falls back to the channel name for old saved entries
This commit is contained in:
Jonathan Sykes
2026-10-02 22:38:05 +08:00
parent 1f9a97a6ae
commit 11ddb8a970
6 changed files with 134 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
server/server.js
View File

@@ -371,6 +371,22 @@ function channelUrlCandidates(c) {
return out;
}
// Name → channel id: the first search hit published by a channel with exactly
// that name (case-insensitive). "<Artist> - Topic" names come from YouTube Music.
async function channelIdForName(name) {
const want = name.trim().toLowerCase();
const out = await runYtdlpResilient([`ytsearch10:${name}`, '--dump-json', '--flat-playlist', '--no-warnings', '--ignore-errors'], { pooled: true });
for (const line of out.split('\n')) {
try {
const j = JSON.parse(line);
const ch = String(j.channel || j.uploader || '').trim().toLowerCase();
const id = j.channel_id || j.uploader_id;
if (ch === want && /^UC[\w-]{22}$/.test(id || '')) return id;
} catch { /* not JSON */ }
}
return null;
}
// Resolve a channel identifier to a /videos URL yt-dlp can fetch
function channelToUrl(c) {
let base = c.trim();
@@ -517,6 +533,21 @@ app.get('/api/channel', async (c) => {
if (results.length) break;
} catch (err) { lastErr = err; }
}
// A bare name ("Maverick City Music - Topic") is not a handle: find the
// channel id through a search, then try its uploads/home again.
if (!results.length && !/^(https?:|@|UC[\w-]{22}$)/.test(chan)) {
const id = await channelIdForName(chan).catch(() => null);
if (id) {
for (const url of channelUrlCandidates(id)) {
try {
out = await runYtdlpResilient([url, '--dump-json', '--flat-playlist', '--no-warnings', '--ignore-errors',
'--playlist-end', String(CHANNEL_LIMIT)], { pooled: true });
results = parseCards(out);
if (results.length) { lastErr = null; break; }
} catch (err) { lastErr = err; }
}
}
}
if (!results.length && lastErr) throw lastErr;
// Extract channel name + URL from the first record
const first = results[0];
@@ -2050,6 +2081,8 @@ const uploads = registerUploadRoutes(app, {
ffmpeg: FFMPEG,
ffprobe: FFPROBE,
uploadDir: UPLOAD_DIR,
backupDir: process.env.UPLOAD_BACKUP_DIR || null,
volumeMarker: process.env.UPLOAD_VOLUME_MARKER || null,
rangeFileResponse,
requireAdminOrToken: notes.requireAdminOrToken,
parseLrc,

46
server/uploads.js
View File

@@ -82,10 +82,28 @@ export function describeProbe(probe, fallbackName) {
export function registerUploadRoutes(app, deps) {
const { db, ffmpeg = 'ffmpeg', ffprobe = 'ffprobe', uploadDir, rangeFileResponse, requireAdminOrToken, parseLrc, sanitizeLyrics } = deps;
mkdirSync(uploadDir, { recursive: true });
const filePath = (u) => join(uploadDir, `${u.id}.${u.ext}`);
const artPath = (id) => join(uploadDir, `${id}.art.jpg`);
// uploadDir may sit on a removable drive (volumeMarker = a file that only
// exists on it). backupDir is the nightly second copy on the stable disk:
// reads fall back to it, and while the drive is offline new uploads land
// there (the nightly sync copies them back over).
const { backupDir = null, volumeMarker = null } = deps;
const online = () => { if (!volumeMarker) return true; try { return existsSync(volumeMarker); } catch { return false; } };
const writeDir = () => {
const d = online() || !backupDir ? uploadDir : backupDir;
try { mkdirSync(d, { recursive: true }); } catch { /* reported by the write itself */ }
return d;
};
const dirs = () => [online() ? uploadDir : null, backupDir].filter(Boolean);
if (online()) mkdirSync(uploadDir, { recursive: true });
if (backupDir) mkdirSync(backupDir, { recursive: true });
const locate = (name) => {
for (const d of dirs()) { const p = join(d, name); if (existsSync(p)) return p; }
return join(writeDir(), name);
};
const filePath = (u) => locate(`${u.id}.${u.ext}`);
const artPath = (id) => locate(`${id}.art.jpg`);
const newArtPath = (id) => join(writeDir(), `${id}.art.jpg`);
const removeEverywhere = (name) => { for (const d of [uploadDir, backupDir].filter(Boolean)) { try { unlinkSync(join(d, name)); } catch { /* gone */ } } };
async function probeFile(path) {
const out = await run(ffprobe, ['-v', 'error', '-print_format', 'json', '-show_format', '-show_streams', path]);
@@ -102,17 +120,17 @@ export function registerUploadRoutes(app, deps) {
// Cover art: an uploaded image wins, else the embedded picture.
let art = null;
if (artFile && typeof artFile !== 'string' && artFile.size) {
const tmp = join(uploadDir, `${id}.art.src`);
const tmp = join(writeDir(), `${id}.art.src`);
await Bun.write(tmp, artFile);
try {
await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", artPath(id)]);
await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", newArtPath(id)]);
art = 'file';
} catch { /* unusable image — ignore */ }
try { unlinkSync(tmp); } catch { /* gone */ }
}
if (!art && info.coverIndex !== null) {
try {
await run(ffmpeg, ['-v', 'error', '-y', '-i', target, '-map', `0:${info.coverIndex}`, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", artPath(id)]);
await run(ffmpeg, ['-v', 'error', '-y', '-i', target, '-map', `0:${info.coverIndex}`, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", newArtPath(id)]);
art = 'embedded';
} catch { /* cover we can't decode */ }
}
@@ -144,7 +162,7 @@ export function registerUploadRoutes(app, deps) {
const extError = (ext) => `unsupported file type ".${ext}" — video: ${[...VIDEO_EXT].join(', ')}; audio: ${[...AUDIO_EXT].join(', ')}`;
const cleanup = (id, target) => {
try { unlinkSync(target); } catch { /* not written */ }
try { unlinkSync(artPath(id)); } catch { /* none */ }
removeEverywhere(`${id}.art.jpg`);
};
// Multipart upload — fine for small files and scripts, but the whole body is
@@ -158,7 +176,7 @@ export function registerUploadRoutes(app, deps) {
const ext = extOf(file.name);
if (!VIDEO_EXT.has(ext) && !AUDIO_EXT.has(ext)) return c.json({ ok: false, error: extError(ext) }, 415);
const id = 'upl_' + randomBytes(6).toString('hex');
const target = join(uploadDir, `${id}.${ext}`);
const target = join(writeDir(), `${id}.${ext}`);
try {
await Bun.write(target, file);
const r = await ingest({ id, ext, target, size: file.size, filename: file.name, fields: body, artFile: body.art });
@@ -181,7 +199,7 @@ export function registerUploadRoutes(app, deps) {
const reader = c.req.raw.body ? c.req.raw.body.getReader() : null;
if (!reader) return c.json({ ok: false, error: 'empty body' }, 400);
const id = 'upl_' + randomBytes(6).toString('hex');
const target = join(uploadDir, `${id}.${ext}`);
const target = join(writeDir(), `${id}.${ext}`);
try {
let got = 0;
const out = createWriteStream(target);
@@ -213,10 +231,10 @@ export function registerUploadRoutes(app, deps) {
if (!UPLOAD_ID_RE.test(id) || !(await db.getUpload(id))) return c.json({ ok: false, error: 'not found' }, 404);
const buf = await c.req.arrayBuffer();
if (!buf.byteLength || buf.byteLength > 20 * 1024 * 1024) return c.json({ ok: false, error: 'send an image under 20 MB' }, 400);
const tmp = join(uploadDir, `${id}.art.src`);
const tmp = join(writeDir(), `${id}.art.src`);
try {
await Bun.write(tmp, buf);
await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", artPath(id)]);
await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", newArtPath(id)]);
await db.setUploadArt(id, 'file');
return c.json({ ok: true });
} catch (err) {
@@ -232,8 +250,8 @@ export function registerUploadRoutes(app, deps) {
const u = await db.getUpload(id);
if (!u) return c.json({ ok: false, error: 'not found' }, 404);
await db.deleteUpload(id);
try { unlinkSync(filePath(u)); } catch { /* gone */ }
try { unlinkSync(artPath(id)); } catch { /* none */ }
removeEverywhere(`${u.id}.${u.ext}`);
removeEverywhere(`${id}.art.jpg`);
return c.json({ ok: true });
});

27
server/uploads.test.js
View File

@@ -165,3 +165,30 @@ describe('streaming upload (large / long files)', () => {
expect((await db.getUpload(up.upload.id)).art).toBeTruthy();
});
});
describe('upload storage on a removable drive', () => {
test('reads the drive copy, falls back to the backup copy, and writes to the backup while offline', async () => {
const { mkdtempSync, writeFileSync, rmSync } = await import('node:fs');
const { tmpdir } = await import('node:os');
const { join } = await import('node:path');
const { registerUploadRoutes } = await import('./uploads.js');
const root = mkdtempSync(join(tmpdir(), 'ytp-up-'));
const drive = join(root, 'usb'), uploadDir = join(drive, 'uploads'), backupDir = join(root, 'backup');
const marker = join(drive, '.hl-data');
(await import('node:fs')).mkdirSync(drive, { recursive: true });
writeFileSync(marker, '');
const noop = () => {};
const app = new Proxy({}, { get: () => noop });
const up = registerUploadRoutes(app, { db: {}, uploadDir, backupDir, volumeMarker: marker, rangeFileResponse: noop, requireAdminOrToken: noop });
const u = { id: 'up_test1', ext: 'mp4' };
writeFileSync(join(backupDir, 'up_test1.mp4'), 'backup');
expect(up.filePath(u)).toBe(join(backupDir, 'up_test1.mp4')); // drive copy missing → backup
writeFileSync(join(uploadDir, 'up_test1.mp4'), 'drive');
expect(up.filePath(u)).toBe(join(uploadDir, 'up_test1.mp4')); // drive copy preferred
rmSync(marker);
expect(up.filePath(u)).toBe(join(backupDir, 'up_test1.mp4')); // drive offline → backup
expect(up.filePath({ id: 'up_new', ext: 'mp4' })).toBe(join(backupDir, 'up_new.mp4')); // new writes go to the stable disk
rmSync(root, { recursive: true, force: true });
});
});