josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extend video catalog source metadata

Browse Source
This commit is contained in:
Jonathan Sykes
2026-10-14 01:22:32 +08:00
parent f465cc86d0
commit 593ecaecd9
7 changed files with 136 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
server/video-catalog.js
View File

@@ -7,6 +7,8 @@ const SOURCES = new Set(['search', 'search-cache', 'client-search', 'channel', '
const text = (v, n = 300) => typeof v === 'string' ? v.trim().slice(0, n) : '';
const positive = v => Number.isFinite(Number(v)) && Number(v) > 0 ? Number(v) : 0;
const canonicalThumb = id => `https://i.ytimg.com/vi/${id}/hqdefault.jpg`;
const canonicalUrl = id => `https://www.youtube.com/watch?v=${id}`;
export const foldSearchText = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
export const channelKey = c => String(c.channelId || c.channel || '').toLowerCase().replace(/\s*-\s*topic$|vevo$|\s+official$/g, '').trim();
function safeThumbnail(url) {
try { const u = new URL(url); return u.protocol === 'https:' && /^(?:i|i\d)\.ytimg\.com$/.test(u.hostname) && !u.port && !u.username && !u.password; }
@@ -23,14 +25,16 @@ export function normalizeCard(v) {
// Never fetch arbitrary client URLs (including redirects or private hosts).
if (safeThumbnail(u.href)) thumbnail = u.href;
} catch { /* canonical YouTube art */ }
const card = { id: v.id, title, channel: text(v.channel), channelId: text(v.channelId, 64),
const card = { id: v.id, title, url: canonicalUrl(v.id), channel: text(v.channel), channelId: text(v.channelId ?? v.channel_id, 64),
duration: positive(v.duration), thumbnail };
try {
const u = new URL(v.channelUrl);
const u = new URL(v.channelUrl || v.channel_url);
if (u.protocol === 'https:' && /^(www\.)?youtube\.com$/.test(u.hostname)) card.channelUrl = u.href;
} catch { /* optional */ }
for (const key of ['tags', 'categories']) if (Array.isArray(v[key])) card[key] = v[key].slice(0, 30).map(x => text(x, 80)).filter(Boolean);
if (v.description) card.description = text(v.description, 1200);
if (v.published || v.publishedAt || v.upload_date) card.published = text(v.published || v.publishedAt || v.upload_date, 40);
if (v.lang || v.language) card.lang = text(v.lang || v.language, 24);
if (positive(v.viewCount ?? v.view_count)) card.viewCount = positive(v.viewCount ?? v.view_count);
return card;
}
@@ -67,9 +71,12 @@ async function ingestBatch(cards, source, fillMissing) {
await db.batch(batch.flatMap(c => {
// Sparse playlist cards must not erase richer extractor metadata.
const merged = fillMissing ? mergeMissing(old.get(c.id) || {}, c) : { ...(old.get(c.id) || {}), ...Object.fromEntries(Object.entries(c).filter(([, v]) => v !== '' && v !== 0 && (!Array.isArray(v) || v.length))) };
return [{ sql: `INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,1,?)
ON CONFLICT(id) DO UPDATE SET card=excluded.card,hay=excluded.hay,seen=seen+1,updated_at=excluded.updated_at`,
args: [c.id, JSON.stringify(merged), `${merged.title} ${merged.channel || ''} ${(merged.tags || []).join(' ')}`.toLowerCase(), now] },
const hay = foldSearchText([merged.title, merged.channel, merged.channelId, merged.description,
...(merged.tags || []), ...(merged.categories || [])].filter(Boolean).join(' '));
return [{ sql: `INSERT INTO video_meta (id,card,hay,seen,updated_at,url,channel_id,channel_url,published,lang) VALUES (?,?,?,1,?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET card=excluded.card,hay=excluded.hay,seen=seen+1,updated_at=excluded.updated_at,
url=excluded.url,channel_id=excluded.channel_id,channel_url=excluded.channel_url,published=excluded.published,lang=excluded.lang`,
args: [c.id, JSON.stringify(merged), hay, now, merged.url, merged.channelId || null, merged.channelUrl || null, merged.published || null, merged.lang || null] },
{ sql: `INSERT INTO video_meta_sources (video_id,source,last_seen) VALUES (?,?,?)
ON CONFLICT(video_id,source) DO UPDATE SET discoveries=discoveries+1,last_seen=excluded.last_seen`, args: [c.id, source, now] },
{ sql: `INSERT INTO video_thumbnails (video_id,url) VALUES (?,?) ON CONFLICT(video_id) DO UPDATE SET retry_at=0