diff --git a/frontend/app.js b/frontend/app.js index 2f1e14e..ce72c94 100755 --- a/frontend/app.js +++ b/frontend/app.js @@ -142,8 +142,8 @@ const RecentSearches = (() => { }, clear() { save([]); }, // saved results stay in SearchLibrary (cleared only from Settings) // Cache the just-fetched result set for `q` (called on every successful search). - cacheResults(q, results) { SearchLibrary.put(q, results); }, - // Cached {query, results, ts} for `q`, or null. + cacheResults(q, results, ts) { SearchLibrary.put(q, results, ts); }, + // Promise of the cached {query, results, ts} for `q`, or null. getResults(q) { return SearchLibrary.get(q); }, @@ -162,62 +162,108 @@ const RecentSearches = (() => { // The thumbnails themselves live in the service worker's `ytplayer-thumbs` // cache (warmed below), so a saved search renders with its pictures offline. const SearchLibrary = (() => { - const KEY = 'ytpSearchLibrary'; - const LEGACY = 'ytpRecentSearchResults'; - const MAX_QUERIES = 150; - const PER_QUERY = 50; + const INDEX_KEY = 'ytpSearchIndex'; // small, synchronous: who/when/how big + const OLD_KEYS = ['ytpSearchLibrary', 'ytpRecentSearchResults']; // earlier localStorage stores + const MAX_QUERIES = 1000; + const PER_QUERY = 100; + const FRESH_MS = 15 * 24 * 3600_000; // a saved copy is trusted for 15 days const norm = (q) => String(q || '').trim().toLowerCase(); - let mem = null; - function read() { - if (mem) return mem; - let o = {}; - try { o = JSON.parse(localStorage.getItem(KEY) || '{}') || {}; } catch { o = {}; } - // One-time import of the old per-recent-search cache. + + // Metadata index { key: { query, count, ts, bytes, thumb } } lives in memory and + // localStorage (≈250 KB at 1000 searches); the result lists themselves — up to + // 100 cards each — live in IndexedDB and are read one search at a time. + let index = {}; + try { index = JSON.parse(localStorage.getItem(INDEX_KEY) || '{}') || {}; } catch { index = {}; } + let persistTimer = 0; + const persistIndex = () => { + clearTimeout(persistTimer); + persistTimer = setTimeout(() => { try { localStorage.setItem(INDEX_KEY, JSON.stringify(index)); } catch { /* quota */ } }, 300); + }; + + // IndexedDB, with a memory fallback when it is blocked (private windows). + const fallback = new Map(); + let dbp = null; + const open = () => dbp || (dbp = new Promise((resolve) => { try { - const old = JSON.parse(localStorage.getItem(LEGACY) || 'null'); - if (old && typeof old === 'object') { - for (const [k, v] of Object.entries(old)) if (v && Array.isArray(v.results) && !o[k]) o[k] = v; - localStorage.removeItem(LEGACY); - } - } catch { /* ignore */ } - mem = o; - return mem; + const req = indexedDB.open('ytpSearchLibrary', 1); + req.onupgradeneeded = () => req.result.createObjectStore('r', { keyPath: 'key' }); + req.onsuccess = () => resolve(req.result); + req.onerror = req.onblocked = () => resolve(null); + } catch { resolve(null); } + })); + async function tx(mode, fn) { + const d = await open(); + if (!d) return fn(null); + return new Promise((resolve) => { + let out; + try { + const t = d.transaction('r', mode); + out = fn(t.objectStore('r')); + t.oncomplete = () => resolve(out && out.result !== undefined ? out.result : undefined); + t.onerror = t.onabort = () => resolve(undefined); + } catch { resolve(undefined); } + }); } - function write() { - const o = read(); - for (let attempt = 0; attempt < 4; attempt++) { - try { localStorage.setItem(KEY, JSON.stringify(o)); return true; } catch { - // Quota: drop the oldest fifth and retry. - const keys = Object.keys(o).sort((a, b) => (o[a].ts || 0) - (o[b].ts || 0)); - keys.slice(0, Math.max(1, Math.ceil(keys.length / 5))).forEach((k) => delete o[k]); + const idbPut = (rec) => tx('readwrite', (st) => st ? st.put(rec) : void fallback.set(rec.key, rec)); + const idbGet = (key) => tx('readonly', (st) => st ? st.get(key) : { result: fallback.get(key) }); + const idbDel = (keys) => tx('readwrite', (st) => { if (st) keys.forEach((k) => st.delete(k)); else keys.forEach((k) => fallback.delete(k)); }); + const idbClear = () => tx('readwrite', (st) => st ? st.clear() : void fallback.clear()); + + // One-time import of the earlier localStorage stores. + (async () => { + for (const k of OLD_KEYS) { + let o = null; + try { o = JSON.parse(localStorage.getItem(k) || 'null'); } catch { o = null; } + if (o && typeof o === 'object') { + for (const [key, v] of Object.entries(o)) { + if (!v || !Array.isArray(v.results) || !v.results.length || index[key]) continue; + put(v.query || key, v.results, v.ts); + } } + try { localStorage.removeItem(k); } catch { /* blocked */ } } - return false; + })(); + + function put(q, results, ts) { + q = String(q || '').trim(); + if (!q || !Array.isArray(results) || !results.length) return; + const key = norm(q); + const list = results.slice(0, PER_QUERY); + const bytes = JSON.stringify(list).length; + index[key] = { query: q, count: list.length, ts: ts || Date.now(), bytes, thumb: (list[0] || {}).thumbnail || '' }; + idbPut({ key, query: q, results: list, ts: index[key].ts }); + const keys = Object.keys(index); + if (keys.length > MAX_QUERIES) { + const drop = keys.sort((a, b) => (index[a].ts || 0) - (index[b].ts || 0)).slice(0, keys.length - MAX_QUERIES); + drop.forEach((k) => delete index[k]); + idbDel(drop); + } + persistIndex(); } return { - put(q, results) { - q = String(q || '').trim(); - if (!q || !Array.isArray(results) || !results.length) return; - const o = read(); - o[norm(q)] = { query: q, results: results.slice(0, PER_QUERY), ts: Date.now() }; - const keys = Object.keys(o); - if (keys.length > MAX_QUERIES) { - keys.sort((a, b) => (o[a].ts || 0) - (o[b].ts || 0)).slice(0, keys.length - MAX_QUERIES).forEach((k) => delete o[k]); - } - write(); + FRESH_MS, + PER_QUERY, + put, + // Promise of { query, results, ts } or null. + async get(q) { + const key = norm(q); + if (!index[key]) return null; + const rec = await idbGet(key); + if (rec && Array.isArray(rec.results) && rec.results.length) return rec; + delete index[key]; persistIndex(); // index entry without data + return null; }, - get(q) { const h = read()[norm(q)]; return h && Array.isArray(h.results) && h.results.length ? h : null; }, - has(q) { return !!read()[norm(q)]; }, - // Newest first: [{ query, count, ts, thumb }] + has(q) { return !!index[norm(q)]; }, + isFresh(ts) { return Date.now() - (ts || 0) < FRESH_MS; }, + // Newest first: [{ key, query, count, ts, thumb }] list() { - const o = read(); - return Object.keys(o).map((k) => ({ key: k, query: o[k].query || k, count: (o[k].results || []).length, ts: o[k].ts || 0, thumb: ((o[k].results || [])[0] || {}).thumbnail || '' })) + return Object.keys(index).map((k) => ({ key: k, query: index[k].query || k, count: index[k].count || 0, ts: index[k].ts || 0, thumb: index[k].thumb || '' })) .sort((a, b) => b.ts - a.ts); }, - remove(keys) { const o = read(); (keys || []).forEach((k) => delete o[k]); write(); }, - clear() { mem = {}; try { localStorage.removeItem(KEY); } catch { /* blocked */ } }, - bytes() { try { return (localStorage.getItem(KEY) || '').length; } catch { return 0; } }, - count() { return Object.keys(read()).length; }, + remove(keys) { (keys || []).forEach((k) => delete index[k]); idbDel(keys || []); persistIndex(); }, + clear() { index = {}; idbClear(); persistIndex(); }, + bytes() { return Object.values(index).reduce((n, x) => n + (x.bytes || 0), 0); }, + count() { return Object.keys(index).length; }, }; })(); @@ -343,9 +389,13 @@ async function opfsClear() { let _currentBlobUrl = null; const API = { - search: (query) => WEB - ? webFetch(`/api/search?q=${encodeURIComponent(query)}`) + search: (query, { refresh = false } = {}) => WEB + ? webFetch(`/api/search?q=${encodeURIComponent(query)}${refresh ? '&refresh=1' : ''}`) : call('yt.search', 'yt_search', { query }), + // Videos the server already knows — answers fast while the real search loads. + searchLocal: (query) => WEB + ? webFetch(`/api/search/local?q=${encodeURIComponent(query)}`) + : Promise.resolve({ ok: false }), getChannel: (channel) => WEB ? webFetch(`/api/channel?c=${encodeURIComponent(channel)}`) : call('yt.channel', 'yt_channel', { channel }), @@ -1303,7 +1353,7 @@ function startCacheResyncWatch() { // Pull a result set's artwork into the service worker's thumbnail cache, a few at // a time, so a saved search still has pictures offline. function warmSearchThumbs(results) { - const urls = (results || []).map((r) => r && r.thumbnail).filter(Boolean).slice(0, 50); + const urls = (results || []).map((r) => r && r.thumbnail).filter(Boolean).slice(0, 100); let i = 0; const next = () => { if (i >= urls.length) return; warmThumb(urls[i++]); setTimeout(next, 40); }; next(); next(); next(); @@ -7476,6 +7526,7 @@ function setListPaneHidden(hidden) { function renderList() { { const ls = $('listSub'); if (ls) ls.textContent = ''; const ps = $('p2pStrip'); if (ps) ps.classList.add('hidden'); } + if (view.type !== 'search') { const sm = $('searchMeta'); if (sm) sm.classList.add('hidden'); } setListPaneHidden(view.type === 'search' && !searchResults.length && !searchAttempted); if (view.type === 'settings') { renderSettings(); return; } if (view.type === 'saved') { renderSaved(); return; } @@ -7878,7 +7929,7 @@ function renderSavedSearches() { row.querySelector('.card-title').textContent = it.query; row.addEventListener('click', () => { els.searchInput.value = it.query; - if (els.runSearch) els.runSearch(it.query, { instant: SearchLibrary.get(it.query) }); + if (els.runSearch) els.runSearch(it.query); }); c.appendChild(row); }); @@ -9535,7 +9586,11 @@ function wireUI() { // `searchSeq` guards against a slower response overwriting a newer one // (the instant paint above can itself be "in flight" behind a background // refresh when the user searches something else right after). - async function runSearchQuery(q, { instant = null } = {}) { + // The saved copy of `q` is read from IndexedDB. A copy younger than 15 days is + // shown as is (no network); an older one is shown at once and refreshed behind + // it. `refresh` (the "Update search" button) always asks the server for fresh + // results, bypassing its caches too. + async function runSearchQuery(q, { refresh = false } = {}) { q = (q || '').trim(); if (!q) return; const mySeq = ++searchSeq; @@ -9543,10 +9598,16 @@ function wireUI() { RecentSearches.add(q); hideRecentSearches(); view = { type: 'search' }; + const instant = refresh ? null : await RecentSearches.getResults(q); + if (mySeq !== searchSeq) return; if (instant) { searchResults = instant.results; searchAttempted = true; render(); + showSearchMeta(q, instant.ts); + // Even on a fresh copy, make sure thumbnails are in the SW cache. + warmSearchThumbs(instant.results); + if (SearchLibrary.isFresh(instant.ts)) { scrollListIntoViewPortrait(); return; } } else { render(); showSearchSkeletons(); @@ -9554,27 +9615,60 @@ function wireUI() { // Even when already on the search view (no view change for render() to // detect), a new query means the user wants to see the results. scrollListIntoViewPortrait(); + const metaBtn = $('searchUpdateBtn'); + if (metaBtn) { metaBtn.disabled = true; metaBtn.textContent = 'Updating…'; } + let networkDone = false, paintedLocal = false; + if (!instant && !refresh) { + // Known videos from the server's metadata store, painted until YouTube answers. + API.searchLocal(q).then((lr) => { + if (networkDone || mySeq !== searchSeq || !lr || !lr.ok || !(lr.results || []).length) return; + paintedLocal = true; + searchResults = lr.results; + searchAttempted = true; + renderList(); + }).catch(() => {}); + } try { - const res = await API.search(q); + const res = await API.search(q, { refresh }); + networkDone = true; if (mySeq !== searchSeq) return; // superseded by a newer search if (!res || !res.ok) throw new Error(res?.error || 'Search failed'); const results = res.results || []; searchResults = results; searchAttempted = true; renderList(); - RecentSearches.cacheResults(q, results); + const ts = Date.now(); + RecentSearches.cacheResults(q, results, ts); + showSearchMeta(q, ts); warmSearchThumbs(results); warmStreams(results.slice(0, 3).map((r) => r.id)); + if (refresh) toast('Search updated'); perfMeasure('ytp:search', 'ytp:search-start'); } catch (err) { + networkDone = true; if (mySeq !== searchSeq) return; - if (instant) return; // keep showing the cached results; fail silently + if (paintedLocal) return; // keep the server's known videos + if (instant) { showSearchMeta(q, instant.ts); if (refresh) toast('Could not update: ' + err.message); return; } // keep the saved copy els.cards.innerHTML = ''; els.status.classList.remove('hidden'); els.status.textContent = '⚠ ' + err.message; } } + // "Saved copy · 3 d ago [↻ Update search]" above the results. + function showSearchMeta(q, ts) { + const m = $('searchMeta'); + if (!m || view.type !== 'search' || !searchResults.length) return; + const left = Math.max(0, Math.ceil((SearchLibrary.FRESH_MS - (Date.now() - ts)) / 86400000)); + m.innerHTML = `Updated ${fmtAgo(ts)} · ${searchResults.length} results${SearchLibrary.isFresh(ts) ? ` · kept ${left} more day${left === 1 ? '' : 's'}` : ' · refreshing'}` + + ''; + m.classList.remove('hidden'); + m.dataset.q = q; + } + $('searchMeta').addEventListener('click', (e) => { + if (e.target.closest('#searchUpdateBtn')) runSearchQuery($('searchMeta').dataset.q || '', { refresh: true }); + }); + els.runSearch = runSearchQuery; els.searchForm.addEventListener('submit', (e) => { e.preventDefault(); @@ -9635,7 +9729,7 @@ function wireUI() { e.preventDefault(); const q = row.dataset.q; els.searchInput.value = q; - runSearchQuery(q, { instant: RecentSearches.getResults(q) }); + runSearchQuery(q); } }); document.addEventListener('click', (e) => { diff --git a/frontend/index.html b/frontend/index.html index 85facd4..15705c1 100755 --- a/frontend/index.html +++ b/frontend/index.html @@ -373,6 +373,7 @@ +
diff --git a/frontend/refresh.css b/frontend/refresh.css index 4065d45..cf18eb0 100644 --- a/frontend/refresh.css +++ b/frontend/refresh.css @@ -545,3 +545,9 @@ .ss-row small { color: var(--text-dim); flex: none; font-size: 11px; } .search-recent-clear small { opacity: 0.6; } .topbar { position: relative; z-index: 80; } /* keeps the recent-searches dropdown above the page content */ + +/* Search results: age line + "Update search" */ +.search-meta { display: flex; align-items: center; justify-content: space-between; gap: 10px; margin: 0 0 10px; padding: 8px 12px; border: 1px solid var(--line); border-radius: 12px; font-size: 12px; color: var(--text-dim); } +.search-meta.hidden { display: none; } +.search-meta .sm-btn { flex: none; border: 1px solid var(--line); background: transparent; color: var(--text); border-radius: 999px; padding: 6px 12px; font: inherit; font-weight: 600; cursor: pointer; } +.search-meta .sm-btn:disabled { opacity: .55; cursor: default; } diff --git a/server/db.js b/server/db.js index 2cfdbdb..ac7d160 100644 --- a/server/db.js +++ b/server/db.js @@ -123,6 +123,17 @@ export async function initDb() { ); CREATE INDEX IF NOT EXISTS idx_search_cache_lru ON search_cache (last_access); + -- Every video the server has ever seen in a search: lets a search show + -- known videos at once while YouTube is still loading. + CREATE TABLE IF NOT EXISTS video_meta ( + id TEXT PRIMARY KEY, + card TEXT NOT NULL, -- the result card, JSON + hay TEXT NOT NULL, -- lowercased title + channel, for matching + seen INTEGER NOT NULL DEFAULT 1, -- times it appeared in a search + updated_at INTEGER NOT NULL DEFAULT 0 + ); + CREATE INDEX IF NOT EXISTS idx_video_meta_upd ON video_meta (updated_at); + -- Shared per-video documents (kind = lyrics | chapters), visible to every -- user. The live copy is here; every save also lands in video_note_revs -- as a full snapshot, which is the server-side backup and undo history. diff --git a/server/search-cache.js b/server/search-cache.js index 7fc5472..49b07dd 100644 --- a/server/search-cache.js +++ b/server/search-cache.js @@ -58,3 +58,38 @@ export async function stats() { const t = (await db.execute('SELECT COUNT(*) AS n, COALESCE(SUM(size),0) AS b FROM search_cache')).rows[0]; return { queries: Number(t.n), bytes: Number(t.b), maxQueries: MAX_ROWS, maxBytes: MAX_BYTES }; } + +// ---- Known-video metadata: instant results while YouTube loads ---- +export const MAX_VIDEOS = Number(process.env.VIDEO_META_MAX) || 500_000; + +export async function rememberVideos(cards) { + const now = Date.now(); + const rows = (cards || []).filter((c) => c && typeof c.id === 'string' && c.title); + if (!rows.length) return; + await db.batch(rows.map((c) => ({ + sql: `INSERT INTO video_meta (id, card, hay, seen, updated_at) VALUES (?,?,?,1,?) + ON CONFLICT(id) DO UPDATE SET card=excluded.card, hay=excluded.hay, seen=seen+1, updated_at=excluded.updated_at`, + args: [c.id, JSON.stringify(c), `${c.title} ${c.channel || ''}`.toLowerCase(), now], + }))); + if (Math.random() < 0.02) trimVideos().catch(() => {}); +} + +// Cards whose title/channel contain EVERY word of the query, most-seen first. +export async function searchVideos(q, limit = 60) { + const words = String(q).toLowerCase().split(/\s+/).filter(Boolean).slice(0, 8); + if (!words.length) return []; + const like = (w) => '%' + w.replace(/[\\%_]/g, '\\$&') + '%'; + const r = await db.execute({ + sql: `SELECT card FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')} + ORDER BY seen DESC, updated_at DESC LIMIT ?`, + args: [...words.map(like), limit], + }); + return r.rows.map((x) => { try { return JSON.parse(x.card); } catch { return null; } }).filter(Boolean); +} + +export async function trimVideos(max = MAX_VIDEOS) { + const n = Number((await db.execute('SELECT COUNT(*) AS n FROM video_meta')).rows[0].n); + if (n <= max) return 0; + await db.execute({ sql: 'DELETE FROM video_meta WHERE id IN (SELECT id FROM video_meta ORDER BY updated_at ASC LIMIT ?)', args: [n - max] }); + return n - max; +} diff --git a/server/search-cache.test.js b/server/search-cache.test.js index 7bafb5b..47a4033 100644 --- a/server/search-cache.test.js +++ b/server/search-cache.test.js @@ -18,3 +18,14 @@ test('trim evicts least-recently-used rows over the row cap', async () => { expect(dropped).toBeGreaterThan(0); expect((await sc.stats()).queries).toBe(3); }); + +test('known videos are remembered and found by every word', async () => { + await sc.rememberVideos([ + { id: 'a1', title: 'Way Maker - Sinach', channel: 'Worship' }, + { id: 'b2', title: 'Way Maker (live)', channel: 'Other' }, + { id: 'c3', title: 'Goodness of God', channel: 'Worship' }, + ]); + expect((await sc.searchVideos('way maker')).map((c) => c.id).sort()).toEqual(['a1', 'b2']); + expect((await sc.searchVideos('maker worship')).map((c) => c.id)).toEqual(['a1']); + expect(await sc.searchVideos('100%')).toEqual([]); +}); diff --git a/server/server.js b/server/server.js index 5f3d40d..e2c6e24 100644 --- a/server/server.js +++ b/server/server.js @@ -428,6 +428,7 @@ function fetchYoutube(q) { const fetchedAt = Date.now(); if (searchCache.size >= SEARCH_CACHE_MAX) searchCache.delete(searchCache.keys().next().value); searchCache.set(key, { yt, fetchedAt }); + if (yt.length) searchCacheDb.rememberVideos(yt).catch(() => {}); if (yt.length) searchCacheDb.put(q, yt).catch((e) => console.warn(`[search] cache write: ${e.message}`)); return { yt, fetchedAt }; })().finally(() => inflightSearches.delete(key)); @@ -440,6 +441,16 @@ const searchLibrary = async (q) => { try { return (await notesDb.listUploads({ q, limit: 20 })).map(uploads.card); } catch { return []; } }; +// GET /api/search/local?q= — videos this server already knows (from every +// earlier search) plus its own uploads; answers in milliseconds so the app can +// show something while the real search is still loading. +app.get('/api/search/local', async (c) => { + const q = (c.req.query('q') || '').trim(); + if (!q) return c.json({ ok: false, error: 'empty query' }, 400); + const [mine, known] = await Promise.all([searchLibrary(q), searchCacheDb.searchVideos(q).catch(() => [])]); + return c.json({ ok: true, results: [...mine, ...known], local: true }); +}); + // GET /api/search?q=[&refresh=1] // Memory → persistent cache (fresh 10 min, then stale-while-revalidate up to 15 // days: answered instantly, refreshed in the background) → YouTube. `refresh=1`