josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add upload library management and ops diagnostics

Browse Source
This commit is contained in:
Jonathan Sykes
2026-10-14 03:09:12 +08:00
parent 5177a4d441
commit 27c694901d
15 changed files with 467 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
server/uploads.js
View File

@@ -23,9 +23,10 @@
* ========================================================================== */
import { spawn } from 'node:child_process';
import { createWriteStream, existsSync, mkdirSync, unlinkSync } from 'node:fs';
import { createWriteStream, existsSync, mkdirSync, unlinkSync, readdirSync, renameSync } from 'node:fs';
import { join } from 'node:path';
import { randomBytes } from 'node:crypto';
import { audit } from './audit-log.js';
export const UPLOAD_ID_RE = /^upl_[a-f0-9]{12}$/;
const MAX_BYTES = 4 * 1024 * 1024 * 1024; // 4 GB
@@ -104,6 +105,7 @@ export function registerUploadRoutes(app, deps) {
const artPath = (id) => locate(`${id}.art.jpg`);
const newArtPath = (id) => join(writeDir(), `${id}.art.jpg`);
const removeEverywhere = (name) => { for (const d of [uploadDir, backupDir].filter(Boolean)) { try { unlinkSync(join(d, name)); } catch { /* gone */ } } };
const removeStaleCopies = (name, currentPath) => { for (const d of [uploadDir, backupDir].filter(Boolean)) { const p = join(d, name); if (p !== currentPath) try { unlinkSync(p); } catch { /* no alternate copy */ } } };
async function probeFile(path) {
const out = await run(ffprobe, ['-v', 'error', '-print_format', 'json', '-show_format', '-show_streams', path]);
@@ -181,6 +183,7 @@ export function registerUploadRoutes(app, deps) {
try {
await Bun.write(target, file);
const r = await ingest({ id, ext, target, size: file.size, filename: file.name, fields: body, artFile: body.art });
await audit(db, 'admin', 'upload.create', id, { size: file.size, kind: r.upload.kind });
return c.json({ ok: true, ...r });
} catch (err) {
cleanup(id, target);
@@ -188,6 +191,85 @@ export function registerUploadRoutes(app, deps) {
}
});
app.get('/api/admin/uploads/manage', requireAdminOrToken, async c => {
const q = String(c.req.query('q') || '').trim().slice(0, 120), offset = Math.max(0, Number(c.req.query('offset')) || 0);
const result = await db.listAdminUploads({ q, offset, limit: 100 });
result.quotas = await db.listUploadQuotas?.() || [];
const known = new Set(await db.listUploadFileNames());
const orphans = [];
for (const dir of dirs()) for (const name of readdirSync(dir).slice(0, 10000)) {
if (!name.includes('.')) continue;
const id = name.split('.')[0];
if (UPLOAD_ID_RE.test(id) && !known.has(name) && !name.includes('.replace.')) orphans.push({ name, dir });
}
return c.json({ ok: true, ...result, offset, orphans: orphans.slice(0, 200) }, 200, { 'Cache-Control': 'no-store' });
});
app.put('/api/admin/uploads/quotas/:owner', requireAdminOrToken, async c => {
const owner = c.req.param('owner'); if (!/^[\w-]{8,128}$/.test(owner)) return c.json({ ok: false, error: 'invalid owner id' }, 400);
const body = await c.req.json().catch(() => ({})), maxBytes = Number(body.maxBytes);
if (!Number.isFinite(maxBytes) || maxBytes < 0 || maxBytes > 10 * 1024 ** 4) return c.json({ ok: false, error: 'quota must be zero (unlimited) or a valid byte count' }, 400);
await db.setUploadQuota?.(owner, maxBytes || null);
await audit(db, 'admin', 'upload.quota.set', owner, { maxBytes: maxBytes || null });
return c.json({ ok: true, maxBytes: maxBytes || null });
});
app.patch('/api/admin/uploads/:id', requireAdminOrToken, async c => {
const id = c.req.param('id'); if (!UPLOAD_ID_RE.test(id)) return c.json({ ok: false, error: 'invalid id' }, 400);
const body = await c.req.json().catch(() => ({})), before = await db.getUpload(id);
if (!before) return c.json({ ok: false, error: 'not found' }, 404);
const patch = {};
for (const key of ['title','artist','album']) if (body[key] !== undefined) patch[key] = String(body[key]).trim();
if (body.listed !== undefined) patch.listed = body.listed === true;
await db.updateUploadMeta(id, patch);
await audit(db, 'admin', 'upload.update', id, patch);
return c.json({ ok: true, upload: await db.getUpload(id) });
});
app.post('/api/admin/uploads/bulk', requireAdminOrToken, async c => {
const body = await c.req.json().catch(() => ({})), ids = [...new Set((Array.isArray(body.ids) ? body.ids : []).filter(id => UPLOAD_ID_RE.test(String(id))))].slice(0, 200);
if (!ids.length || !['delete','list','unlist'].includes(body.action)) return c.json({ ok: false, error: 'choose uploads and delete, list, or unlist' }, 400);
let changed = 0;
for (const id of ids) {
const u = await db.getUpload(id); if (!u) continue;
if (body.action === 'delete') { await db.deleteUpload(id); removeEverywhere(`${u.id}.${u.ext}`); removeEverywhere(`${id}.art.jpg`); }
else await db.updateUploadMeta(id, { listed: body.action === 'list' });
await audit(db, 'admin', `upload.bulk.${body.action}`, id); changed++;
}
return c.json({ ok: true, changed });
});
app.put('/api/admin/uploads/:id/replace', requireAdminOrToken, async c => {
const id = c.req.param('id'), old = UPLOAD_ID_RE.test(id) ? await db.getUpload(id) : null;
if (!old) return c.json({ ok: false, error: 'not found' }, 404);
const filename = String(c.req.query('name') || 'replacement.' + old.ext), ext = extOf(filename);
if (!VIDEO_EXT.has(ext) && !AUDIO_EXT.has(ext)) return c.json({ ok: false, error: extError(ext) }, 415);
const declared = Number(c.req.header('content-length')); if (declared > MAX_BYTES) return c.json({ ok: false, error: 'file is larger than 4 GB' }, 413);
const reader = c.req.raw.body?.getReader(); if (!reader) return c.json({ ok: false, error: 'empty body' }, 400);
const dir = writeDir(), temp = join(dir, `${id}.replace.${randomBytes(4).toString('hex')}.${ext}`), target = join(dir, `${id}.${ext}`);
let got = 0; const out = createWriteStream(temp);
try {
try { for (;;) { const { done, value } = await reader.read(); if (done) break; got += value.byteLength; if (got > MAX_BYTES) throw Object.assign(Error('file is larger than 4 GB'), { status: 413 }); if (!out.write(value)) await new Promise(r => out.once('drain', r)); } }
finally { await new Promise(r => out.end(r)); }
if (!got || (declared > 0 && got !== declared)) throw Object.assign(Error('empty or incomplete replacement'), { status: 400 });
const probe = await probeFile(temp), info = describeProbe(probe, filename);
if (!info.hasAudio) throw Object.assign(Error('replacement has no audio stream'), { status: 415 });
const backup = `${target}.previous`;
const existingTarget = existsSync(target);
if (existingTarget) renameSync(target, backup);
try {
renameSync(temp, target);
await db.replaceUploadFile(id, { kind: info.kind, title: String(c.req.query('title') ?? old.title).slice(0,300), artist: String(c.req.query('artist') ?? old.artist).slice(0,200), album: String(c.req.query('album') ?? old.album).slice(0,200), duration: info.duration, ext, mime: MIME[ext] || (info.kind === 'video' ? 'video/mp4' : 'audio/mpeg'), size: got, art: old.art });
} catch (error) { try { unlinkSync(target); } catch {} if (existingTarget) renameSync(backup, target); throw error; }
if (existingTarget) try { unlinkSync(backup); } catch {}
removeStaleCopies(`${id}.${ext}`, target);
if (old.ext !== ext) removeEverywhere(`${id}.${old.ext}`);
await audit(db, 'admin', 'upload.replace', id, { oldExt: old.ext, ext, size: got });
return c.json({ ok: true, upload: await db.getUpload(id) });
} catch (error) { return c.json({ ok: false, error: error.status ? error.message : 'replacement failed validation' }, error.status || 422); }
finally { try { unlinkSync(temp); } catch {} }
});
// Streaming upload: the raw file is the request body, copied to disk chunk by
// chunk so a multi-GB (hour-long) video never sits in memory.
// PUT /api/admin/uploads/stream?name=talk.mp4&title=&artist=&album=
@@ -219,6 +301,7 @@ export function registerUploadRoutes(app, deps) {
if (declared > 0 && got !== declared) throw Object.assign(new Error(`upload cut short (${got} of ${declared} bytes)`), { status: 400 });
const q = (k) => String(c.req.query(k) || '');
const r = await ingest({ id, ext, target, size: got, filename: name, fields: { title: q('title'), artist: q('artist'), album: q('album') } });
await audit(db, 'admin', 'upload.create', id, { size: got, kind: r.upload.kind });
return c.json({ ok: true, ...r });
} catch (err) {
cleanup(id, target);
@@ -252,6 +335,8 @@ export function registerUploadRoutes(app, deps) {
if (declared > SHARED.fileBytes) return c.json({ ok: false, error: `shared files can be up to ${mb(SHARED.fileBytes)}` }, 413);
if (sharedActive.has(fp)) return c.json({ ok: false, error: 'this device is already uploading — wait for it to finish' }, 429);
const use = await db.sharedUploadUsage(fp);
const ownerQuota = await db.getUploadQuota?.(fp);
if (ownerQuota && use.bytes + declared > ownerQuota) return c.json({ ok: false, error: `owner storage quota of ${mb(ownerQuota)} reached` }, 413);
if (use.today >= SHARED.perDay) return c.json({ ok: false, error: `limit of ${SHARED.perDay} shared uploads a day reached` }, 429);
if (use.bytes + declared > SHARED.deviceBytes) return c.json({ ok: false, error: `this device's ${mb(SHARED.deviceBytes)} of shared uploads is used up` }, 413);
if (use.totalShared + declared > SHARED.totalBytes) return c.json({ ok: false, error: 'the server has no room for more shared uploads' }, 507);
@@ -268,8 +353,8 @@ export function registerUploadRoutes(app, deps) {
const { done, value } = await reader.read();
if (done) break;
got += value.byteLength;
if (got > SHARED.fileBytes || use.bytes + got > SHARED.deviceBytes) {
throw Object.assign(new Error(`shared files can be up to ${mb(Math.min(SHARED.fileBytes, SHARED.deviceBytes - use.bytes))}`), { status: 413 });
if (got > SHARED.fileBytes || use.bytes + got > SHARED.deviceBytes || (ownerQuota && use.bytes + got > ownerQuota)) {
throw Object.assign(new Error(`shared storage quota reached (owner cap ${ownerQuota ? mb(ownerQuota) : mb(SHARED.deviceBytes)})`), { status: 413 });
}
if (!out.write(value)) await new Promise((r) => out.once('drain', r));
}
@@ -306,6 +391,8 @@ export function registerUploadRoutes(app, deps) {
await Bun.write(tmp, buf);
await run(ffmpeg, ['-v', 'error', '-y', '-i', tmp, '-frames:v', '1', '-vf', "scale='min(800,iw)':-2", newArtPath(id)]);
await db.setUploadArt(id, 'file');
removeStaleCopies(`${id}.art.jpg`, newArtPath(id));
await audit(db, 'admin', 'upload.art.replace', id, { size: buf.byteLength });
return c.json({ ok: true });
} catch (err) {
return c.json({ ok: false, error: 'unusable image: ' + err.message }, 400);
@@ -320,6 +407,7 @@ export function registerUploadRoutes(app, deps) {
const u = await db.getUpload(id);
if (!u) return c.json({ ok: false, error: 'not found' }, 404);
await db.deleteUpload(id);
await audit(db, 'admin', 'upload.delete', id, { size: u.size });
removeEverywhere(`${u.id}.${u.ext}`);
removeEverywhere(`${id}.art.jpg`);
return c.json({ ok: true });
@@ -335,6 +423,8 @@ export function registerUploadRoutes(app, deps) {
const id = c.req.param('id');
if (!UPLOAD_ID_RE.test(id)) return c.json({ ok: false, error: 'invalid id' }, 400);
const u = await db.getUpload(id);
const requestedGen = Number(c.req.query('g'));
if (requestedGen && requestedGen !== Number(u?.gen || 1)) return c.json({ ok: false, error: 'upload generation changed' }, 409, { 'Cache-Control': 'no-store' });
if (!u || !existsSync(filePath(u))) return c.json({ ok: false, error: 'not found' }, 404);
db.touchUpload(id);
return rangeFileResponse(c, filePath(u), u.mime, 'public, max-age=31536000, immutable');
@@ -343,6 +433,8 @@ export function registerUploadRoutes(app, deps) {
app.get('/api/uploads/:id/art', async (c) => {
const id = c.req.param('id');
if (!UPLOAD_ID_RE.test(id)) return c.json({ ok: false, error: 'invalid id' }, 400);
const row = await db.getUpload(id), requestedGen = Number(c.req.query('g'));
if (requestedGen && requestedGen !== Number(row?.artGen || 1)) return c.json({ ok: false, error: 'upload artwork generation changed' }, 409, { 'Cache-Control': 'no-store' });
const p = artPath(id);
if (!existsSync(p)) return c.json({ ok: false, error: 'no cover art' }, 404);
return rangeFileResponse(c, p, 'image/jpeg', 'public, max-age=31536000, immutable');
@@ -351,20 +443,20 @@ export function registerUploadRoutes(app, deps) {
// /api/streams payload for an upload: one "Original" quality for video,
// audio + cover art for audio (the app shows the art where the video goes).
function streamsPayload(u) {
const art = u.art ? `/api/uploads/${u.id}/art` : '';
const url = `/api/uploads/${u.id}`;
const art = u.art ? `/api/uploads/${u.id}/art?g=${u.artGen || 1}` : '';
const url = `/api/uploads/${u.id}?g=${u.gen || 1}`;
return {
meta: { id: u.id, title: u.title, channel: u.artist || u.album || 'Uploaded', channelId: '', channelUrl: '', duration: u.duration, thumbnail: art },
audioUrl: u.kind === 'audio' ? url : '',
qualities: u.kind === 'video' ? [{ label: 'Original', height: 0, hasAudio: true, url }] : [],
upload: true,
upload: true, gen: u.gen || 1,
kind: u.kind,
art,
};
}
const card = (u) => ({
id: u.id, title: u.title, channel: u.artist || u.album || 'Uploaded', channelId: '', channelUrl: '',
duration: u.duration, thumbnail: u.art ? `/api/uploads/${u.id}/art` : '', upload: true, kind: u.kind,
id: u.id, title: u.title, channel: u.artist || u.album || 'Uploaded', channelId: '', channelUrl: '', gen: u.gen || 1,
duration: u.duration, thumbnail: u.art ? `/api/uploads/${u.id}/art?g=${u.artGen || 1}` : '', upload: true, kind: u.kind,
});
return { streamsPayload, card, filePath, artPath, isUploadId: (id) => UPLOAD_ID_RE.test(id) };