87 lines
3.7 KiB
JavaScript
87 lines
3.7 KiB
JavaScript
// Persistent search cache (libsql). Holds the YouTube cards of a search so a
|
|
// repeat query — from any device, or after a restart — never waits on YouTube.
|
|
// Capped by row count (default 250,000 queries) AND bytes (default 2 GiB), LRU.
|
|
import { db } from './db.js';
|
|
import { ingest, withLocalThumbnails, trimCatalog } from './video-catalog.js';
|
|
|
|
export const MAX_ROWS = Number(process.env.SEARCH_CACHE_MAX_ROWS) || 250_000;
|
|
export const MAX_BYTES = Number(process.env.SEARCH_CACHE_MAX_BYTES) || 2 * 1024 ** 3;
|
|
export const PER_QUERY = 200;
|
|
|
|
let writes = 0;
|
|
|
|
export async function get(q) {
|
|
const key = q.toLowerCase();
|
|
const r = await db.execute({ sql: 'SELECT results, fetched_at FROM search_cache WHERE q = ?', args: [key] });
|
|
const row = r.rows[0];
|
|
if (!row) return null;
|
|
let results;
|
|
try { results = JSON.parse(row.results); } catch { return null; }
|
|
// Touch at most once a minute per row — reads must stay cheap.
|
|
db.execute({ sql: 'UPDATE search_cache SET last_access = ? WHERE q = ? AND last_access < ?',
|
|
args: [Date.now(), key, Date.now() - 60_000] }).catch(() => {});
|
|
return { results, fetchedAt: Number(row.fetched_at) };
|
|
}
|
|
|
|
export async function put(q, results) {
|
|
const json = JSON.stringify(results.slice(0, PER_QUERY));
|
|
const now = Date.now();
|
|
await db.execute({
|
|
sql: `INSERT INTO search_cache (q, results, size, fetched_at, last_access) VALUES (?,?,?,?,?)
|
|
ON CONFLICT(q) DO UPDATE SET results=excluded.results, size=excluded.size,
|
|
fetched_at=excluded.fetched_at, last_access=excluded.last_access`,
|
|
args: [q.toLowerCase(), json, json.length, now, now],
|
|
});
|
|
if (++writes % 25 === 0) trim().catch(() => {});
|
|
}
|
|
|
|
// Drop least-recently-used rows until under both budgets.
|
|
export async function trim({ maxRows = MAX_ROWS, maxBytes = MAX_BYTES } = {}) {
|
|
const t = (await db.execute('SELECT COUNT(*) AS n, COALESCE(SUM(size),0) AS b FROM search_cache')).rows[0];
|
|
let n = Number(t.n), b = Number(t.b);
|
|
if (n <= maxRows && b <= maxBytes) return 0;
|
|
let dropped = 0;
|
|
while (n > maxRows || b > maxBytes) {
|
|
const batch = (await db.execute('SELECT q, size FROM search_cache ORDER BY last_access ASC LIMIT 500')).rows;
|
|
if (!batch.length) break;
|
|
const drop = [];
|
|
for (const r of batch) {
|
|
if (n <= maxRows && b <= maxBytes) break;
|
|
drop.push(r.q); n -= 1; b -= Number(r.size);
|
|
}
|
|
await db.batch(drop.map((q) => ({ sql: 'DELETE FROM search_cache WHERE q = ?', args: [q] })));
|
|
dropped += drop.length;
|
|
}
|
|
return dropped;
|
|
}
|
|
|
|
export async function stats() {
|
|
const t = (await db.execute('SELECT COUNT(*) AS n, COALESCE(SUM(size),0) AS b FROM search_cache')).rows[0];
|
|
return { queries: Number(t.n), bytes: Number(t.b), maxQueries: MAX_ROWS, maxBytes: MAX_BYTES };
|
|
}
|
|
|
|
// ---- Known-video metadata: instant results while YouTube loads ----
|
|
export const MAX_VIDEOS = Number(process.env.VIDEO_META_MAX) || 500_000;
|
|
|
|
export async function rememberVideos(cards) {
|
|
await ingest(cards, 'search');
|
|
if (Math.random() < 0.02) trimVideos().catch(() => {});
|
|
}
|
|
|
|
// Cards whose title/channel contain EVERY word of the query, most-seen first.
|
|
export async function searchVideos(q, limit = 60) {
|
|
const words = String(q).toLowerCase().split(/\s+/).filter(Boolean).slice(0, 8);
|
|
if (!words.length) return [];
|
|
const like = (w) => '%' + w.replace(/[\\%_]/g, '\\$&') + '%';
|
|
const r = await db.execute({
|
|
sql: `SELECT card FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')}
|
|
ORDER BY seen DESC, updated_at DESC LIMIT ?`,
|
|
args: [...words.map(like), limit],
|
|
});
|
|
return withLocalThumbnails(r.rows.map((x) => { try { return JSON.parse(x.card); } catch { return null; } }).filter(Boolean));
|
|
}
|
|
|
|
export async function trimVideos(max = MAX_VIDEOS) {
|
|
return trimCatalog(max);
|
|
}
|