josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge codex/admin-ops: editor redesign, deep search, admin tools and ops

Browse Source
This commit is contained in:
Jonathan Sykes
2026-10-15 11:42:07 +08:00
parent 313d5aef99 27c694901d
commit 59871a5b88
83 changed files with 3463 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
server/server.js
View File

@@ -47,6 +47,7 @@ import { initDb, upsertUser, recordVideoAccess, getUserData, createProfile, getP
import { createMediaCache, HIGH, LOW, validateMedia, MediaSkip } from './media-cache.js';
import * as notesDb from './db.js';
import { registerNoteRoutes, parseLrc, sanitizeLyrics } from './notes.js';
import { registerAdminToolsRoutes } from './admin-tools.js';
import { createRemoteHub } from './remote.js';
import { createPartyHub } from './party.js';
import { registerMediaMetaRoutes } from './media-meta.js';
@@ -63,12 +64,15 @@ import { sha256Range } from './hash.js';
import * as innertube from './innertube.js';
import { createRelatedService, registerRelatedRoute } from './related.js';
import * as searchCacheDb from './search-cache.js';
import { ingest as collectVideoMetadata, syncListening, linkListening, startThumbnails } from './video-catalog.js';
import { ingest as collectVideoMetadata, syncListening, linkListening, startThumbnails, backfillStep } from './video-catalog.js';
import { registerCatalogRoutes } from './recommendations.js';
import QRCode from 'qrcode';
import { createYtdlpPool } from './ytdlp-pool.js';
import { createWarmQueue } from './warm-queue.js';
import { registerDeepSearchRoutes } from './deep-search.js';
import { dirname, join as pathJoin } from 'node:path';
import { createRequestMonitor, registerRequestMonitorRoutes } from './request-monitor.js';
import { registerP2pAdminRoutes } from './p2p-admin.js';
// A media proxy must not die because one client's stream hit an edge case
// (see /api/play cancel()): log and keep serving instead of crash-looping.
@@ -268,6 +272,7 @@ function slimEntry(j) {
return {
id,
title: pick(j, 'title') || '(untitled)',
url: /^[\w-]{11}$/.test(id) ? `https://www.youtube.com/watch?v=${id}` : null,
channel: pickChannel(j),
channelId: pickChannelId(j),
channelUrl: pickChannelUrl(j),
@@ -343,6 +348,22 @@ function channelToUrl(c) {
const app = new Hono();
app.use('*', logger());
const requestMonitor = createRequestMonitor({
snapshotFile: pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'ops-metrics.jsonl'),
snapshotIntervalMs: process.env.OPS_MONITOR === '0' ? 0 : 60_000,
diskPaths: [{ name: 'app-data', path: dirname(process.env.DB_PATH || './data/ytplayer.db') }, { name: 'media', path: process.env.MEDIA_DIR || './data/media' }],
adapters: {
ytdlp: () => ytdlpPool ? ytdlpPool.stats() : null,
media: () => media?.stats ? media.stats() : null,
transcription: async () => {
try { const rows = (await notesDb.db.execute("SELECT status,COUNT(*) AS n FROM lyric_transcriptions WHERE status IN ('queued','running') GROUP BY status")).rows;
const counts = Object.fromEntries(rows.map(row => [row.status, Number(row.n)])); return { queued: counts.queued || 0, active: counts.running || 0 };
} catch { return null; }
},
cache: () => ({ ...searchCacheCounters, memoryEntries: searchCache.size }),
},
});
if (process.env.OPS_MONITOR !== '0') app.use('*', requestMonitor.middleware());
registerCatalogRoutes(app, { resolveListener: recommendationListener });
// ============================================================================
@@ -380,6 +401,7 @@ const SEARCH_CACHE_TTL_MS = 10 * 60_000; // fresh: answered from memory
const SEARCH_CACHE_STALE_MS = 15 * 24 * 3600_000; // persistent copy: stale-while-revalidate for 15 days
const searchCache = new Map(); // lowercased q -> { yt, fetchedAt } (YouTube cards only)
const inflightSearches = new Map(); // lowercased q -> Promise<{ yt, youtubeError? }>
const searchCacheCounters = { hits: 0, misses: 0 };
// The YouTube side of one search (up to 200 cards), shared by every caller asking
// the same thing at the same time, and written to memory + the persistent cache.
@@ -431,6 +453,8 @@ const searchLibrary = async (q) => {
// This server's own library first — it still answers when YouTube is unreachable.
try { return (await notesDb.listUploads({ q, limit: 20, listedOnly: true })).map(uploads.card); } catch { return []; }
};
const withYoutubeUrl = card => card && /^[\w-]{11}$/.test(card.id || '')
? { ...card, url: `https://www.youtube.com/watch?v=${card.id}` } : card;
// GET /api/search/local?q=<query> — videos this server already knows (from every
// earlier search) plus its own uploads; answers in milliseconds so the app can
@@ -439,7 +463,7 @@ app.get('/api/search/local', async (c) => {
const q = (c.req.query('q') || '').trim();
if (!q) return c.json({ ok: false, error: 'empty query' }, 400);
const [mine, known] = await Promise.all([searchLibrary(q), searchCacheDb.searchVideos(q).catch(() => [])]);
return c.json({ ok: true, results: [...mine, ...known], local: true });
return c.json({ ok: true, results: [...mine, ...known].map(withYoutubeUrl), local: true });
});
// GET /api/search?q=<query>[&refresh=1]
@@ -453,7 +477,7 @@ app.get('/api/search', async (c) => {
const key = q.toLowerCase();
const mine = await searchLibrary(q);
const answer = (yt, extra = {}) => {
const body = JSON.stringify({ ok: true, results: [...mine, ...yt], ...extra });
const body = JSON.stringify({ ok: true, results: [...mine, ...yt].map(withYoutubeUrl), ...extra });
const headers = { 'Content-Type': 'application/json; charset=utf-8', Vary: 'Accept-Encoding' };
if (/\bgzip\b/.test(c.req.header('accept-encoding') || '') && body.length > 1024)
return new Response(Bun.gzipSync(body), { headers: { ...headers, 'Content-Encoding': 'gzip' } });
@@ -466,11 +490,13 @@ app.get('/api/search', async (c) => {
if (row) { hit = { yt: row.results, fetchedAt: row.fetchedAt }; searchCache.set(key, hit); }
}
const age = hit ? Date.now() - hit.fetchedAt : Infinity;
if (hit && age < SEARCH_CACHE_TTL_MS) return answer(hit.yt, { fetchedAt: hit.fetchedAt });
if (hit && age < SEARCH_CACHE_TTL_MS) { searchCacheCounters.hits++; return answer(hit.yt, { fetchedAt: hit.fetchedAt }); }
if (hit && age < SEARCH_CACHE_STALE_MS) {
searchCacheCounters.hits++;
fetchYoutube(q).catch(() => { /* keep serving the stale copy */ });
return answer(hit.yt, { stale: true, fetchedAt: hit.fetchedAt });
}
searchCacheCounters.misses++;
}
try {
const { yt, fetchedAt, partial } = await fetchYoutube(q);
@@ -1121,7 +1147,7 @@ const SAVE_DEFAULT_FORMAT =
// again) share ONE yt-dlp run instead of spawning another each time.
const inflightDownloads = new Map(); // key -> { promise, waiters, tmpBase }
async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) {
async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal, sourceUrl = null) {
await assertNotLive(videoId);
const key = videoId + '|' + formatArgs.join(' ');
let entry = inflightDownloads.get(key);
@@ -1130,10 +1156,11 @@ async function ytdlpDownloadResponse(videoId, fp, formatArgs, signal) {
const tmp = `${tmpdir()}/${tmpBase}.mp4`;
const ctl = new AbortController();
entry = { waiters: 0, tmpBase, ctl };
entry.promise = withSaveSlot(() => {
entry.promise = withSaveSlot(async () => {
if (ctl.signal.aborted) throw new Error('save cancelled');
const url = sourceUrl || await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`;
return runYtdlpResilient([
`https://www.youtube.com/watch?v=${videoId}`,
url,
'--no-warnings', '--no-playlist',
...formatArgs,
'--limit-rate', DOWNLOAD_RATE,
@@ -1203,14 +1230,16 @@ async function ytdlpEditedDownloadResponse(videoId, fp, keep, signal, cachedSrc
try {
// 1) Grab the full source (video+audio merged) so ffmpeg has both streams
// — unless the server cache already holds it.
if (!cachedSrc) await withSaveSlot(() => runYtdlpResilient([
`https://www.youtube.com/watch?v=${videoId}`,
'--no-warnings', '--no-playlist',
'-f', SAVE_MUX_FORMAT,
'--merge-output-format', 'mp4',
'--limit-rate', DOWNLOAD_RATE,
'-o', srcTmp,
], { signal }));
if (!cachedSrc) await withSaveSlot(async () => {
const url = await videoCatalog.getSourceUrl(videoId) || `https://www.youtube.com/watch?v=${videoId}`;
return runYtdlpResilient([
url, '--no-warnings', '--no-playlist',
'-f', SAVE_MUX_FORMAT,
'--merge-output-format', 'mp4',
'--limit-rate', DOWNLOAD_RATE,
'-o', srcTmp,
], { signal });
});
// 2) Trim + concat the keep segments into the final custom video.
await runFfmpeg([
'-y', '-hide_banner', '-loglevel', 'error',
@@ -1260,6 +1289,7 @@ if (!MEDIA_VOLUME_MARKER || existsSync(MEDIA_VOLUME_MARKER)) mkdirSync(MEDIA_DIR
const media = createMediaCache({
dir: MEDIA_DIR,
volumeMarker: MEDIA_VOLUME_MARKER,
getSourceUrl: videoCatalog.getSourceUrl,
db: {
getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats,
// Retention (docs/p2p-architecture.md flow 9): cold copies go before popular ones.
@@ -1268,9 +1298,9 @@ const media = createMediaCache({
}),
},
getInfo: async (videoId) => (await resolveStreams(videoId)).info,
download: async (videoId, out) => {
download: async (videoId, out, sourceUrl) => {
await withSaveSlot(() => runYtdlpResilient([
`https://www.youtube.com/watch?v=${videoId}`,
sourceUrl || `https://www.youtube.com/watch?v=${videoId}`,
'--no-warnings', '--no-playlist',
'-f', SAVE_MUX_FORMAT,
'--merge-output-format', 'mp4',
@@ -2315,6 +2345,8 @@ app.get('/api/remote/qr/:code', async (c) => {
// ============================================================================
const notes = registerNoteRoutes(app, {
db: notesDb,
catalog: videoCatalog,
media,
getProfile,
profileNameRe: PROFILE_NAME_RE,
runYtdlp: runYtdlpResilient,
@@ -2328,6 +2360,12 @@ const notes = registerNoteRoutes(app, {
{ label: 'Database', path: dirname(process.env.DB_PATH || './data/ytplayer.db') },
] },
});
const deepSearch = registerDeepSearchRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, autoStart: false,
searchDeep: innertube.searchDeep, related: innertube.related, browseUploads: innertube.browseUploads,
ingest: collectVideoMetadata, putSearch: searchCacheDb.put });
registerAdminToolsRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, media,
backupDir: process.env.UPLOAD_BACKUP_DIR || pathJoin(dirname(process.env.DB_PATH || './data/ytplayer.db'), 'backups'), backfillStep,
pullLyrics: notes.pullLyricsForAdmin, queueWhisper: notes.queueWhisperForAdmin });
// ============================================================================
// Uploads — the server's own searchable media library (see uploads.js)
@@ -2347,6 +2385,7 @@ const uploads = registerUploadRoutes(app, {
parseLrc,
sanitizeLyrics,
});
registerRequestMonitorRoutes(app, { adminAuth: notes.requireAdminOrToken, monitor: requestMonitor });
const isUpload = (id) => uploads.isUploadId(id);
// ============================================================================
@@ -2372,6 +2411,8 @@ const p2pHub = createP2pHub({ getDevice: p2pDb.getDevice, enabled: () => P2P.ena
directProfileFails.set(key, { count: old && Date.now() - old.at < 900000 ? old.count + 1 : 1, at: Date.now() });
if (directProfileFails.size > 5000) directProfileFails.clear(); return '';
} });
registerP2pAdminRoutes(app, { db: notesDb, adminAuth: notes.requireAdminOrToken, p2pDb, hub: p2pHub,
stats: async () => ({ intakeOpen: intake.openTickets(), intakeActive: intake.active(), content: (await p2pDb.p2pStats()).content }) });
// GET /api/p2p/holders?v=<videoId>|cid=<sha256> — who holds a copy. Rows are
// persistent; `stale` flags a holder not re-verified for P2P_STALE_DAYS.
@@ -2539,6 +2580,7 @@ app.get('/*', indexHtml);
// ============================================================================
async function main() {
await initDb();
deepSearch.start();
startThumbnails();
await p2pDb.initP2pSchema();
console.log(`[ytplayer] DB ready`);