157 lines
10 KiB
JavaScript
157 lines
10 KiB
JavaScript
import { db } from './db.js';
|
|
import { ingest, extractCards, withLocalThumbnails, channelKey, syncListening } from './video-catalog.js';
|
|
|
|
const tokens = s => new Set(String(s || '').toLowerCase().match(/[\p{L}\p{N}]{3,}/gu)?.filter(w => !['the', 'and', 'for', 'with', 'official', 'video', 'music', 'audio', 'lyrics', 'live'].includes(w)) || []);
|
|
const channelOf = channelKey;
|
|
|
|
export function rankRecommendations(candidates, { limit = 12 } = {}) {
|
|
const seeds = candidates.filter(c => c.personal > 0).sort((a, b) => b.personal - a.personal).slice(0, 20);
|
|
const channels = new Map(), interests = new Map();
|
|
for (const c of seeds) {
|
|
const weight = Math.log2(1 + c.personal);
|
|
const channel = channelOf(c.card);
|
|
if (channel) channels.set(channel, (channels.get(channel) || 0) + weight);
|
|
for (const w of tokens(`${c.card.title} ${(c.card.tags || []).join(' ')} ${(c.card.categories || []).join(' ')}`)) interests.set(w, (interests.get(w) || 0) + weight);
|
|
}
|
|
const ranked = candidates.map(c => {
|
|
const words = tokens(`${c.card.title} ${(c.card.tags || []).join(' ')} ${(c.card.categories || []).join(' ')}`);
|
|
let overlap = 0; for (const w of words) overlap += interests.get(w) || 0;
|
|
overlap /= Math.max(1, words.size);
|
|
const channel = channels.get(channelOf(c.card)) || 0;
|
|
const popularity = Math.log2(1 + c.plays);
|
|
const score = 3 * Math.log2(1 + c.personal) + 2 * Math.log2(1 + channel) + Math.log2(1 + overlap)
|
|
+ popularity + 0.6 * Math.log2(1 + c.recent);
|
|
const reason = c.personal > 0 ? 'One of your most played' : channel > 0 ? `More from ${c.card.channel || 'a channel you enjoy'}`
|
|
: overlap > 0 ? 'Similar to your most played' : c.plays > 0 ? 'Popular with listeners' : 'Recently discovered';
|
|
return { ...c.card, reason, score, personal: c.personal, plays: c.plays };
|
|
}).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
|
|
const result = [], channelCounts = new Map();
|
|
let familiar = 0;
|
|
// Reserve room for discoveries when the catalog has them, and vary channels.
|
|
const hasDiscovery = ranked.some(c => !c.personal);
|
|
for (const card of ranked) {
|
|
const channel = channelOf(card);
|
|
if (channel && (channelCounts.get(channel) || 0) >= 3) continue;
|
|
if (hasDiscovery && card.personal && familiar >= Math.ceil(limit / 2)) continue;
|
|
result.push(card); if (card.personal) familiar++;
|
|
if (channel) channelCounts.set(channel, (channelCounts.get(channel) || 0) + 1);
|
|
if (result.length >= limit) break;
|
|
}
|
|
return result.map(({ score, personal, plays, ...card }) => card);
|
|
}
|
|
|
|
export async function recommend(fingerprint = '', limit = 12) {
|
|
const recent = new Date(Date.now() - 30 * 86400000).toISOString().slice(0, 10);
|
|
// Popular/personal seeds plus recent discoveries; no media_cache join/filter.
|
|
const rows = (await db.execute({ sql: `WITH popular AS (
|
|
SELECT video_id FROM listening_daily GROUP BY video_id ORDER BY SUM(plays) DESC LIMIT 300
|
|
), personal AS (
|
|
SELECT video_id FROM listening_daily WHERE fingerprint=? GROUP BY video_id ORDER BY SUM(plays) DESC LIMIT 100
|
|
), discovered AS (
|
|
SELECT id AS video_id FROM video_meta ORDER BY updated_at DESC LIMIT 600
|
|
), ids AS (
|
|
SELECT video_id FROM popular UNION SELECT video_id FROM personal UNION SELECT video_id FROM discovered
|
|
), plays AS (
|
|
SELECT video_id,SUM(plays) AS plays,
|
|
SUM(CASE WHEN fingerprint=? THEN plays ELSE 0 END) AS personal,
|
|
SUM(CASE WHEN day>=? THEN plays ELSE 0 END) AS recent
|
|
FROM listening_daily WHERE video_id IN (SELECT video_id FROM ids) GROUP BY video_id
|
|
) SELECT m.card,m.hay,COALESCE(p.plays,0) AS plays,COALESCE(p.personal,0) AS personal,
|
|
COALESCE(p.recent,0) AS recent FROM video_meta m LEFT JOIN plays p ON p.video_id=m.id
|
|
WHERE m.id IN (SELECT video_id FROM ids)`, args: [fingerprint, fingerprint, recent] })).rows;
|
|
const candidates = rows.map(r => ({ ...r, card: JSON.parse(r.card), plays: Number(r.plays), personal: Number(r.personal), recent: Number(r.recent) }));
|
|
// Include older, unplayed videos from seed channels, even in a large catalog.
|
|
const seedChannels = [...new Set(candidates.filter(c => c.personal > 0).sort((a, b) => b.personal - a.personal).slice(0, 10).map(c => channelOf(c.card)).filter(Boolean))];
|
|
if (seedChannels.length) {
|
|
const extra = (await db.execute({ sql: `WITH nearby AS (
|
|
${seedChannels.map(() => 'SELECT video_id FROM (SELECT video_id FROM video_channels WHERE channel=? ORDER BY updated_at DESC LIMIT 30)').join(' UNION ')}
|
|
) SELECT m.card,COALESCE(SUM(l.plays),0) AS plays,
|
|
COALESCE(SUM(CASE WHEN l.fingerprint=? THEN l.plays ELSE 0 END),0) AS personal,
|
|
COALESCE(SUM(CASE WHEN l.day>=? THEN l.plays ELSE 0 END),0) AS recent
|
|
FROM nearby n JOIN video_meta m ON m.id=n.video_id LEFT JOIN listening_daily l ON l.video_id=n.video_id GROUP BY m.id`, args: [...seedChannels, fingerprint, recent] })).rows;
|
|
const known = new Set(candidates.map(c => c.card.id));
|
|
for (const r of extra) { const card = JSON.parse(r.card); if (!known.has(card.id)) { known.add(card.id); candidates.push({ card, plays: Number(r.plays), personal: Number(r.personal), recent: Number(r.recent) }); } }
|
|
}
|
|
return withLocalThumbnails(rankRecommendations(candidates, { limit }));
|
|
}
|
|
|
|
// Capture cards from successful discovery responses, including future routes.
|
|
// Incoming profile/sync payloads also carry cards absent from the response.
|
|
export function registerCatalogRoutes(app, { resolveListener = async (c, name, fp) => name ? c.json({ ok: false, error: 'profile unavailable' }, 401) : fp,
|
|
adminAuth = async c => c.json({ ok: false, error: 'admin unavailable' }, 503) } = {}) {
|
|
// Anonymous submissions: a per-IP budget, and they may only add videos or fill blanks.
|
|
const collectLog = new Map();
|
|
const overCollectBudget = c => {
|
|
const ip = (c.req.header('cf-connecting-ip') || c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || '').split(',')[0].trim() || 'local';
|
|
const now = Date.now(), list = (collectLog.get(ip) || []).filter(t => now - t < 10 * 60_000);
|
|
list.push(now); collectLog.set(ip, list);
|
|
if (collectLog.size > 5000) collectLog.clear();
|
|
return list.length > 30;
|
|
};
|
|
app.use('/api/*', async (c, next) => {
|
|
const path = c.req.path;
|
|
const incoming = c.req.method === 'POST' && /^\/api\/(user\/sync|profile\/(create|save)|playlist\/share)$/.test(path)
|
|
? c.req.raw.clone() : null;
|
|
await next();
|
|
if (!c.res.ok) return;
|
|
const discovery = c.req.method === 'GET' && !/^\/api\/admin\/(analytics|metadata|collections)(\/|$)/.test(path) && !path.startsWith('/api/catalog/') && path !== '/api/recommendations';
|
|
try {
|
|
if (discovery && c.res.headers.get('content-type')?.includes('application/json')) {
|
|
// /api/search answers gzipped; read the body decompressed.
|
|
const res = c.res.clone();
|
|
const result = res.headers.get('content-encoding') === 'gzip'
|
|
? JSON.parse(new TextDecoder().decode(Bun.gunzipSync(new Uint8Array(await res.arrayBuffer()))))
|
|
: await res.json();
|
|
if (typeof result.data === 'string') { try { result.data = JSON.parse(result.data); } catch { /* invalid profile */ } }
|
|
await ingest(extractCards(result), path.includes('search') ? 'search-cache' : path.includes('channel') ? 'channel' : path.includes('streams') ? 'streams' : 'playlist');
|
|
}
|
|
if (incoming) {
|
|
const body = await incoming.json();
|
|
const payload = typeof body.data === 'string' ? JSON.parse(body.data) : body;
|
|
await ingest(extractCards(payload), path.includes('profile') ? 'profile' : 'sync');
|
|
if (path.startsWith('/api/profile/')) {
|
|
const name = body.name || (await c.res.clone().json()).name;
|
|
if (name) await syncListening('profile:' + name.toLowerCase(), body.data?.stats);
|
|
}
|
|
}
|
|
} catch (e) { console.warn('[catalog] collection:', e.message); }
|
|
});
|
|
app.get('/api/recommendations', async c => {
|
|
const fp = String(c.req.query('fp') || '').slice(0, 200);
|
|
const limit = Math.max(1, Math.min(24, Math.floor(Number(c.req.query('limit')) || 12)));
|
|
try {
|
|
const listener = await resolveListener(c, c.req.query('profile') || '', fp);
|
|
if (typeof listener !== 'string') return listener;
|
|
return c.json({ ok: true, results: await recommend(listener, limit) }, 200, { 'Cache-Control': 'no-store' });
|
|
}
|
|
catch (e) { console.warn('[recommendations]', e.message); return c.json({ ok: false, error: 'Recommendations are unavailable right now.' }, 503); }
|
|
});
|
|
async function readCards(c, maxBytes, maxCards) {
|
|
if (Number(c.req.header('content-length')) > maxBytes) return c.json({ ok: false, error: 'metadata too large' }, 413);
|
|
let body;
|
|
try { const raw = await c.req.text(); if (raw.length > maxBytes) return c.json({ ok: false, error: 'metadata too large' }, 413); body = JSON.parse(raw); }
|
|
catch { return c.json({ ok: false, error: 'invalid JSON' }, 400); }
|
|
if (!Array.isArray(body?.cards) || body.cards.length > maxCards) return c.json({ ok: false, error: `expected at most ${maxCards} cards` }, 400);
|
|
return body.cards;
|
|
}
|
|
app.post('/api/catalog/collect', async c => {
|
|
if (overCollectBudget(c)) return c.json({ ok: false, error: 'too many submissions — try again later' }, 429);
|
|
const cards = await readCards(c, 512000, 500);
|
|
if (!Array.isArray(cards)) return cards;
|
|
try { return c.json({ ok: true, collected: await ingest(cards, 'client-search', { fillMissing: true }) }); }
|
|
catch { return c.json({ ok: false, error: 'Could not save metadata.' }, 503); }
|
|
});
|
|
// Bulk import (channel/playlist listings): admin login or API token; may overwrite metadata.
|
|
app.post('/api/admin/catalog/import', adminAuth, async c => {
|
|
const cards = await readCards(c, 4 * 1024 * 1024, 5000);
|
|
if (!Array.isArray(cards)) return cards;
|
|
try { return c.json({ ok: true, collected: await ingest(cards, 'collector') }); }
|
|
catch { return c.json({ ok: false, error: 'Could not save metadata.' }, 503); }
|
|
});
|
|
app.get('/api/catalog/:id/thumbnail', async c => {
|
|
const r = (await db.execute({ sql: 'SELECT data,mime FROM video_thumbnails WHERE video_id=? AND data IS NOT NULL', args: [c.req.param('id')] })).rows[0];
|
|
if (!r) return c.notFound();
|
|
return new Response(r.data, { headers: { 'Content-Type': r.mime, 'Cache-Control': 'public, max-age=86400', 'X-Content-Type-Options': 'nosniff' } });
|
|
});
|
|
}
|