96 lines
4.3 KiB
JavaScript
96 lines
4.3 KiB
JavaScript
// Persistent search cache (libsql). Holds the YouTube cards of a search so a
|
|
// repeat query — from any device, or after a restart — never waits on YouTube.
|
|
// Capped by row count (default 250,000 queries) AND bytes (default 2 GiB), LRU.
|
|
import { db } from './db.js';
|
|
|
|
export const MAX_ROWS = Number(process.env.SEARCH_CACHE_MAX_ROWS) || 250_000;
|
|
export const MAX_BYTES = Number(process.env.SEARCH_CACHE_MAX_BYTES) || 2 * 1024 ** 3;
|
|
export const PER_QUERY = 200;
|
|
|
|
let writes = 0;
|
|
|
|
export async function get(q) {
|
|
const key = q.toLowerCase();
|
|
const r = await db.execute({ sql: 'SELECT results, fetched_at FROM search_cache WHERE q = ?', args: [key] });
|
|
const row = r.rows[0];
|
|
if (!row) return null;
|
|
let results;
|
|
try { results = JSON.parse(row.results); } catch { return null; }
|
|
// Touch at most once a minute per row — reads must stay cheap.
|
|
db.execute({ sql: 'UPDATE search_cache SET last_access = ? WHERE q = ? AND last_access < ?',
|
|
args: [Date.now(), key, Date.now() - 60_000] }).catch(() => {});
|
|
return { results, fetchedAt: Number(row.fetched_at) };
|
|
}
|
|
|
|
export async function put(q, results) {
|
|
const json = JSON.stringify(results.slice(0, PER_QUERY));
|
|
const now = Date.now();
|
|
await db.execute({
|
|
sql: `INSERT INTO search_cache (q, results, size, fetched_at, last_access) VALUES (?,?,?,?,?)
|
|
ON CONFLICT(q) DO UPDATE SET results=excluded.results, size=excluded.size,
|
|
fetched_at=excluded.fetched_at, last_access=excluded.last_access`,
|
|
args: [q.toLowerCase(), json, json.length, now, now],
|
|
});
|
|
if (++writes % 25 === 0) trim().catch(() => {});
|
|
}
|
|
|
|
// Drop least-recently-used rows until under both budgets.
|
|
export async function trim({ maxRows = MAX_ROWS, maxBytes = MAX_BYTES } = {}) {
|
|
const t = (await db.execute('SELECT COUNT(*) AS n, COALESCE(SUM(size),0) AS b FROM search_cache')).rows[0];
|
|
let n = Number(t.n), b = Number(t.b);
|
|
if (n <= maxRows && b <= maxBytes) return 0;
|
|
let dropped = 0;
|
|
while (n > maxRows || b > maxBytes) {
|
|
const batch = (await db.execute('SELECT q, size FROM search_cache ORDER BY last_access ASC LIMIT 500')).rows;
|
|
if (!batch.length) break;
|
|
const drop = [];
|
|
for (const r of batch) {
|
|
if (n <= maxRows && b <= maxBytes) break;
|
|
drop.push(r.q); n -= 1; b -= Number(r.size);
|
|
}
|
|
await db.batch(drop.map((q) => ({ sql: 'DELETE FROM search_cache WHERE q = ?', args: [q] })));
|
|
dropped += drop.length;
|
|
}
|
|
return dropped;
|
|
}
|
|
|
|
export async function stats() {
|
|
const t = (await db.execute('SELECT COUNT(*) AS n, COALESCE(SUM(size),0) AS b FROM search_cache')).rows[0];
|
|
return { queries: Number(t.n), bytes: Number(t.b), maxQueries: MAX_ROWS, maxBytes: MAX_BYTES };
|
|
}
|
|
|
|
// ---- Known-video metadata: instant results while YouTube loads ----
|
|
export const MAX_VIDEOS = Number(process.env.VIDEO_META_MAX) || 500_000;
|
|
|
|
export async function rememberVideos(cards) {
|
|
const now = Date.now();
|
|
const rows = (cards || []).filter((c) => c && typeof c.id === 'string' && c.title);
|
|
if (!rows.length) return;
|
|
await db.batch(rows.map((c) => ({
|
|
sql: `INSERT INTO video_meta (id, card, hay, seen, updated_at) VALUES (?,?,?,1,?)
|
|
ON CONFLICT(id) DO UPDATE SET card=excluded.card, hay=excluded.hay, seen=seen+1, updated_at=excluded.updated_at`,
|
|
args: [c.id, JSON.stringify(c), `${c.title} ${c.channel || ''}`.toLowerCase(), now],
|
|
})));
|
|
if (Math.random() < 0.02) trimVideos().catch(() => {});
|
|
}
|
|
|
|
// Cards whose title/channel contain EVERY word of the query, most-seen first.
|
|
export async function searchVideos(q, limit = 60) {
|
|
const words = String(q).toLowerCase().split(/\s+/).filter(Boolean).slice(0, 8);
|
|
if (!words.length) return [];
|
|
const like = (w) => '%' + w.replace(/[\\%_]/g, '\\$&') + '%';
|
|
const r = await db.execute({
|
|
sql: `SELECT card FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')}
|
|
ORDER BY seen DESC, updated_at DESC LIMIT ?`,
|
|
args: [...words.map(like), limit],
|
|
});
|
|
return r.rows.map((x) => { try { return JSON.parse(x.card); } catch { return null; } }).filter(Boolean);
|
|
}
|
|
|
|
export async function trimVideos(max = MAX_VIDEOS) {
|
|
const n = Number((await db.execute('SELECT COUNT(*) AS n FROM video_meta')).rows[0].n);
|
|
if (n <= max) return 0;
|
|
await db.execute({ sql: 'DELETE FROM video_meta WHERE id IN (SELECT id FROM video_meta ORDER BY updated_at ASC LIMIT ?)', args: [n - max] });
|
|
return n - max;
|
|
}
|