josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extend video catalog source metadata

Browse Source
This commit is contained in:
Jonathan Sykes
2026-10-14 01:22:32 +08:00
parent f465cc86d0
commit 593ecaecd9
7 changed files with 136 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
server/db.js
View File

@@ -257,6 +257,7 @@ export async function initDb() {
last_used_at INTEGER
);
`);
await migrateVideoMeta();
// Uploads shared from devices (PWA share target): who sent it, and whether it
// is listed in everyone's search (admin uploads) or reachable only by link.
for (const col of ['owner TEXT', 'listed INTEGER NOT NULL DEFAULT 1']) {
@@ -284,6 +285,47 @@ export async function initDb() {
}
}
const YOUTUBE_ID = /^[\w-]{11}$/;
const folded = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
// These columns were added after video_meta shipped. Keep startup migration
// idempotent and page the backfill so a large catalogue never becomes one
// unbounded JS allocation.
export async function migrateVideoMeta({ pageSize = 500 } = {}) {
for (const [name, type] of [['url', 'TEXT'], ['channel_id', 'TEXT'], ['channel_url', 'TEXT'], ['published', 'TEXT'], ['lang', 'TEXT']]) {
try { await db.execute(`ALTER TABLE video_meta ADD COLUMN ${name} ${type}`); } catch { /* column already exists */ }
}
await db.execute('CREATE TABLE IF NOT EXISTS catalog_schema_migrations (version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL)');
const done = await db.execute({ sql: 'SELECT 1 FROM catalog_schema_migrations WHERE version=1' });
if (done.rows.length) return;
let cursor = '';
for (;;) {
const rows = (await db.execute({
sql: 'SELECT id,card,hay FROM video_meta WHERE id>? ORDER BY id LIMIT ?', args: [cursor, pageSize],
})).rows;
if (!rows.length) break;
const updates = [];
for (const row of rows) {
cursor = row.id;
let card = {};
try { card = JSON.parse(row.card || '{}'); } catch { /* malformed legacy card */ }
const url = YOUTUBE_ID.test(row.id) ? `https://www.youtube.com/watch?v=${row.id}` : null;
const channelId = card.channelId || card.channel_id || null;
const channelUrl = card.channelUrl || card.channel_url || null;
const published = card.published || card.publishedAt || card.upload_date || null;
const lang = card.lang || card.language || null;
card = { ...card, url, channelId: channelId || '', channelUrl: channelUrl || '' };
const hay = folded([card.title, card.channel, channelId, card.description,
...(Array.isArray(card.tags) ? card.tags : []), ...(Array.isArray(card.categories) ? card.categories : [])].filter(Boolean).join(' '));
updates.push({ sql: 'UPDATE video_meta SET card=?,url=?,channel_id=?,channel_url=?,published=?,lang=?,hay=? WHERE id=?',
args: [JSON.stringify(card), url, channelId, channelUrl, published, lang, hay || folded(row.hay), row.id] });
}
await db.batch(updates, 'write');
if (rows.length < pageSize) break;
}
await db.execute({ sql: 'INSERT OR IGNORE INTO catalog_schema_migrations(version,applied_at) VALUES(1,?)', args: [Date.now()] });
}
// ---- Shared video notes (lyrics / chapters) ---------------------------------
function noteRow(r) {

1
server/innertube.js
View File

@@ -20,6 +20,7 @@ function cardsFromSections(sections) {
const path = be.canonicalBaseUrl || (be.browseId ? `/channel/${be.browseId}` : '');
out.push({
id: v.videoId,
url: `https://www.youtube.com/watch?v=${v.videoId}`,
title: v.title?.runs?.map((r) => r.text).join('') || '(untitled)',
channel: owner.text || '',
channelId: be.browseId || '',

23
server/search-cache.js
View File

@@ -2,7 +2,7 @@
// repeat query — from any device, or after a restart — never waits on YouTube.
// Capped by row count (default 250,000 queries) AND bytes (default 2 GiB), LRU.
import { db } from './db.js';
import { ingest, withLocalThumbnails, trimCatalog } from './video-catalog.js';
import { ingest, withLocalThumbnails, trimCatalog, foldSearchText } from './video-catalog.js';
export const MAX_ROWS = Number(process.env.SEARCH_CACHE_MAX_ROWS) || 250_000;
export const MAX_BYTES = Number(process.env.SEARCH_CACHE_MAX_BYTES) || 2 * 1024 ** 3;
@@ -70,15 +70,28 @@ export async function rememberVideos(cards) {
// Cards whose title/channel contain EVERY word of the query, most-seen first.
export async function searchVideos(q, limit = 60) {
const words = String(q).toLowerCase().split(/\s+/).filter(Boolean).slice(0, 8);
const folded = foldSearchText(q).trim();
const words = folded.split(/\s+/).filter(Boolean).slice(0, 8);
if (!words.length) return [];
const like = (w) => '%' + w.replace(/[\\%_]/g, '\\$&') + '%';
const r = await db.execute({
sql: `SELECT card FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')}
sql: `SELECT card,seen FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')}
ORDER BY seen DESC, updated_at DESC LIMIT ?`,
args: [...words.map(like), limit],
args: [...words.map(like), Math.max(limit * 10, 200)],
});
return withLocalThumbnails(r.rows.map((x) => { try { return JSON.parse(x.card); } catch { return null; } }).filter(Boolean));
const titleHasAll = title => words.every(word => foldSearchText(title).includes(word));
const phrase = folded;
const ranked = r.rows.map(row => {
try {
const card = JSON.parse(row.card);
const title = foldSearchText(card.title);
const channel = foldSearchText(card.channel);
const score = (title === phrase || title.includes(phrase) ? 4 : 0) +
(titleHasAll(card.title) ? 2 : 0) + (words.some(word => channel.includes(word)) ? 1 : 0);
return { card: { ...card, url: card.url || `https://www.youtube.com/watch?v=${card.id}` }, score, seen: Number(row.seen) || 0 };
} catch { return null; }
}).filter(Boolean).sort((a, b) => b.score - a.score || b.seen - a.seen).slice(0, limit).map(x => x.card);
return withLocalThumbnails(ranked);
}
export async function trimVideos(max = MAX_VIDEOS) {

10
server/search-cache.test.js
View File

@@ -21,11 +21,11 @@ test('trim evicts least-recently-used rows over the row cap', async () => {
test('known videos are remembered and found by every word', async () => {
await sc.rememberVideos([
{ id: 'a1', title: 'Way Maker - Sinach', channel: 'Worship' },
{ id: 'b2', title: 'Way Maker (live)', channel: 'Other' },
{ id: 'c3', title: 'Goodness of God', channel: 'Worship' },
{ id: 'aaa11111111', title: 'Way Maker - Sinach', channel: 'Worship' },
{ id: 'bbb22222222', title: 'Way Maker (live)', channel: 'Other' },
{ id: 'ccc33333333', title: 'Goodness of God', channel: 'Worship' },
]);
expect((await sc.searchVideos('way maker')).map((c) => c.id).sort()).toEqual(['a1', 'b2']);
expect((await sc.searchVideos('maker worship')).map((c) => c.id)).toEqual(['a1']);
expect((await sc.searchVideos('way maker')).map((c) => c.id).sort()).toEqual(['aaa11111111', 'bbb22222222']);
expect((await sc.searchVideos('maker worship')).map((c) => c.id)).toEqual(['aaa11111111']);
expect(await sc.searchVideos('100%')).toEqual([]);
});

7
server/server.js
View File

@@ -268,6 +268,7 @@ function slimEntry(j) {
return {
id,
title: pick(j, 'title') || '(untitled)',
url: /^[\w-]{11}$/.test(id) ? `https://www.youtube.com/watch?v=${id}` : null,
channel: pickChannel(j),
channelId: pickChannelId(j),
channelUrl: pickChannelUrl(j),
@@ -418,6 +419,8 @@ const searchLibrary = async (q) => {
// This server's own library first — it still answers when YouTube is unreachable.
try { return (await notesDb.listUploads({ q, limit: 20, listedOnly: true })).map(uploads.card); } catch { return []; }
};
const withYoutubeUrl = card => card && /^[\w-]{11}$/.test(card.id || '')
? { ...card, url: `https://www.youtube.com/watch?v=${card.id}` } : card;
// GET /api/search/local?q=<query> — videos this server already knows (from every
// earlier search) plus its own uploads; answers in milliseconds so the app can
@@ -426,7 +429,7 @@ app.get('/api/search/local', async (c) => {
const q = (c.req.query('q') || '').trim();
if (!q) return c.json({ ok: false, error: 'empty query' }, 400);
const [mine, known] = await Promise.all([searchLibrary(q), searchCacheDb.searchVideos(q).catch(() => [])]);
return c.json({ ok: true, results: [...mine, ...known], local: true });
return c.json({ ok: true, results: [...mine, ...known].map(withYoutubeUrl), local: true });
});
// GET /api/search?q=<query>[&refresh=1]
@@ -439,7 +442,7 @@ app.get('/api/search', async (c) => {
if (!q) return c.json({ ok: false, error: 'empty query' }, 400);
const key = q.toLowerCase();
const mine = await searchLibrary(q);
const answer = (yt, extra = {}) => c.json({ ok: true, results: [...mine, ...yt], ...extra });
const answer = (yt, extra = {}) => c.json({ ok: true, results: [...mine, ...yt].map(withYoutubeUrl), ...extra });
if (c.req.query('refresh') !== '1') {
let hit = searchCache.get(key);
if (!hit) {

17
server/video-catalog.js
View File

@@ -7,6 +7,8 @@ const SOURCES = new Set(['search', 'search-cache', 'client-search', 'channel', '
const text = (v, n = 300) => typeof v === 'string' ? v.trim().slice(0, n) : '';
const positive = v => Number.isFinite(Number(v)) && Number(v) > 0 ? Number(v) : 0;
const canonicalThumb = id => `https://i.ytimg.com/vi/${id}/hqdefault.jpg`;
const canonicalUrl = id => `https://www.youtube.com/watch?v=${id}`;
export const foldSearchText = value => String(value || '').normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
export const channelKey = c => String(c.channelId || c.channel || '').toLowerCase().replace(/\s*-\s*topic$|vevo$|\s+official$/g, '').trim();
function safeThumbnail(url) {
try { const u = new URL(url); return u.protocol === 'https:' && /^(?:i|i\d)\.ytimg\.com$/.test(u.hostname) && !u.port && !u.username && !u.password; }
@@ -23,14 +25,16 @@ export function normalizeCard(v) {
// Never fetch arbitrary client URLs (including redirects or private hosts).
if (safeThumbnail(u.href)) thumbnail = u.href;
} catch { /* canonical YouTube art */ }
const card = { id: v.id, title, channel: text(v.channel), channelId: text(v.channelId, 64),
const card = { id: v.id, title, url: canonicalUrl(v.id), channel: text(v.channel), channelId: text(v.channelId ?? v.channel_id, 64),
duration: positive(v.duration), thumbnail };
try {
const u = new URL(v.channelUrl);
const u = new URL(v.channelUrl || v.channel_url);
if (u.protocol === 'https:' && /^(www\.)?youtube\.com$/.test(u.hostname)) card.channelUrl = u.href;
} catch { /* optional */ }
for (const key of ['tags', 'categories']) if (Array.isArray(v[key])) card[key] = v[key].slice(0, 30).map(x => text(x, 80)).filter(Boolean);
if (v.description) card.description = text(v.description, 1200);
if (v.published || v.publishedAt || v.upload_date) card.published = text(v.published || v.publishedAt || v.upload_date, 40);
if (v.lang || v.language) card.lang = text(v.lang || v.language, 24);
if (positive(v.viewCount ?? v.view_count)) card.viewCount = positive(v.viewCount ?? v.view_count);
return card;
}
@@ -67,9 +71,12 @@ async function ingestBatch(cards, source, fillMissing) {
await db.batch(batch.flatMap(c => {
// Sparse playlist cards must not erase richer extractor metadata.
const merged = fillMissing ? mergeMissing(old.get(c.id) || {}, c) : { ...(old.get(c.id) || {}), ...Object.fromEntries(Object.entries(c).filter(([, v]) => v !== '' && v !== 0 && (!Array.isArray(v) || v.length))) };
return [{ sql: `INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,1,?)
ON CONFLICT(id) DO UPDATE SET card=excluded.card,hay=excluded.hay,seen=seen+1,updated_at=excluded.updated_at`,
args: [c.id, JSON.stringify(merged), `${merged.title} ${merged.channel || ''} ${(merged.tags || []).join(' ')}`.toLowerCase(), now] },
const hay = foldSearchText([merged.title, merged.channel, merged.channelId, merged.description,
...(merged.tags || []), ...(merged.categories || [])].filter(Boolean).join(' '));
return [{ sql: `INSERT INTO video_meta (id,card,hay,seen,updated_at,url,channel_id,channel_url,published,lang) VALUES (?,?,?,1,?,?,?,?,?,?)
ON CONFLICT(id) DO UPDATE SET card=excluded.card,hay=excluded.hay,seen=seen+1,updated_at=excluded.updated_at,
url=excluded.url,channel_id=excluded.channel_id,channel_url=excluded.channel_url,published=excluded.published,lang=excluded.lang`,
args: [c.id, JSON.stringify(merged), hay, now, merged.url, merged.channelId || null, merged.channelUrl || null, merged.published || null, merged.lang || null] },
{ sql: `INSERT INTO video_meta_sources (video_id,source,last_seen) VALUES (?,?,?)
ON CONFLICT(video_id,source) DO UPDATE SET discoveries=discoveries+1,last_seen=excluded.last_seen`, args: [c.id, source, now] },
{ sql: `INSERT INTO video_thumbnails (video_id,url) VALUES (?,?) ON CONFLICT(video_id) DO UPDATE SET retry_at=0

53
server/video-catalog.test.js Normal file
View File

@@ -0,0 +1,53 @@
import { test, expect } from 'bun:test';
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
const root = mkdtempSync(join(tmpdir(), 'ytp-video-catalog-'));
process.env.DB_PATH = join(root, 'test.db');
const db = await import('./db.js');
const catalog = await import('./video-catalog.js');
await db.initDb();
test('catalog migration backfills canonical URLs and rich metadata, excluding upload ids', async () => {
await db.db.execute('DELETE FROM catalog_schema_migrations WHERE version=1');
await db.db.execute({ sql: 'INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,?,?)',
args: ['abcDEF12345', JSON.stringify({ id: 'abcDEF12345', title: 'Café Worship', channel: 'Choir', channelId: 'UC123', channelUrl: 'https://www.youtube.com/@choir', tags: ['live'], publishedAt: '2024-01-01', language: 'en' }), 'cafe worship choir', 1, 1] });
await db.db.execute({ sql: 'INSERT INTO video_meta (id,card,hay,seen,updated_at) VALUES (?,?,?,?,?)',
args: ['upl_abcdef12', JSON.stringify({ id: 'upl_abcdef12', title: 'Upload' }), 'upload', 1, 1] });
await db.migrateVideoMeta({ pageSize: 1 });
await db.migrateVideoMeta({ pageSize: 1 });
const youtube = (await db.db.execute("SELECT * FROM video_meta WHERE id='abcDEF12345'")).rows[0];
const upload = (await db.db.execute("SELECT * FROM video_meta WHERE id='upl_abcdef12'")).rows[0];
expect(youtube.url).toBe('https://www.youtube.com/watch?v=abcDEF12345');
expect(youtube.channel_id).toBe('UC123');
expect(youtube.channel_url).toBe('https://www.youtube.com/@choir');
expect(youtube.published).toBe('2024-01-01');
expect(youtube.lang).toBe('en');
expect(youtube.hay).toContain('cafe');
expect(JSON.parse(youtube.card).url).toBe(youtube.url);
expect(upload.url).toBeNull();
});
test('ingest stores canonical source URL and channel metadata on the card and row', async () => {
await catalog.ingest([{ id: 'xyzABC12345', title: 'Música', channel: 'Choir', channel_id: 'UC456', channel_url: 'https://www.youtube.com/@choir', tags: ['Praise'], description: 'Sunday service' }], 'deep');
const row = (await db.db.execute("SELECT * FROM video_meta WHERE id='xyzABC12345'")).rows[0];
const card = JSON.parse(row.card);
expect(row.url).toBe('https://www.youtube.com/watch?v=xyzABC12345');
expect(row.channel_id).toBe('UC456');
expect(card.url).toBe(row.url);
expect(row.hay).toContain('musica');
expect(row.hay).toContain('praise');
});
test('accent-insensitive matching ranks title phrase ahead of channel match', async () => {
await catalog.ingest([
{ id: 'phrase12345', title: 'Café Worship Live', channel: 'Other' },
{ id: 'channel1234', title: 'Sunday Set', channel: 'Cafe Worship Live' },
{ id: 'words123456', title: 'Worship in the Cafe', channel: 'Other' },
], 'deep');
const search = await import('./search-cache.js');
const results = await search.searchVideos('cafe worship live');
expect(results[0].id).toBe('phrase12345');
expect(results[0].url).toBe('https://www.youtube.com/watch?v=phrase12345');
});