IndexedDB search library (1000x100, 15-day copies, Update search), server video metadata for instant local results

frontend/app.js

@@ -142,8 +142,8 @@ const RecentSearches = (() => {
     },
     clear() { save([]); }, // saved results stay in SearchLibrary (cleared only from Settings)
     // Cache the just-fetched result set for `q` (called on every successful search).
-    cacheResults(q, results) { SearchLibrary.put(q, results); },
+    cacheResults(q, results, ts) { SearchLibrary.put(q, results, ts); },
-    // Cached {query, results, ts} for `q`, or null.
+    // Promise of the cached {query, results, ts} for `q`, or null.
     getResults(q) {
       return SearchLibrary.get(q);
     },
@@ -162,62 +162,108 @@ const RecentSearches = (() => {
 // The thumbnails themselves live in the service worker's `ytplayer-thumbs`
 // cache (warmed below), so a saved search renders with its pictures offline.
 const SearchLibrary = (() => {
-  const KEY = 'ytpSearchLibrary';
+  const INDEX_KEY = 'ytpSearchIndex';      // small, synchronous: who/when/how big
-  const LEGACY = 'ytpRecentSearchResults';
+  const OLD_KEYS = ['ytpSearchLibrary', 'ytpRecentSearchResults']; // earlier localStorage stores
-  const MAX_QUERIES = 150;
+  const MAX_QUERIES = 1000;
-  const PER_QUERY = 50;
+  const PER_QUERY = 100;
+  const FRESH_MS = 15 * 24 * 3600_000;     // a saved copy is trusted for 15 days
   const norm = (q) => String(q || '').trim().toLowerCase();
-  let mem = null;
+
-  function read() {
+  // Metadata index { key: { query, count, ts, bytes, thumb } } lives in memory and
-    if (mem) return mem;
+  // localStorage (≈250 KB at 1000 searches); the result lists themselves — up to
-    let o = {};
+  // 100 cards each — live in IndexedDB and are read one search at a time.
-    try { o = JSON.parse(localStorage.getItem(KEY) || '{}') || {}; } catch { o = {}; }
+  let index = {};
-    // One-time import of the old per-recent-search cache.
+  try { index = JSON.parse(localStorage.getItem(INDEX_KEY) || '{}') || {}; } catch { index = {}; }
+  let persistTimer = 0;
+  const persistIndex = () => {
+    clearTimeout(persistTimer);
+    persistTimer = setTimeout(() => { try { localStorage.setItem(INDEX_KEY, JSON.stringify(index)); } catch { /* quota */ } }, 300);
+  };
+
+  // IndexedDB, with a memory fallback when it is blocked (private windows).
+  const fallback = new Map();
+  let dbp = null;
+  const open = () => dbp || (dbp = new Promise((resolve) => {
     try {
-      const old = JSON.parse(localStorage.getItem(LEGACY) || 'null');
+      const req = indexedDB.open('ytpSearchLibrary', 1);
-      if (old && typeof old === 'object') {
+      req.onupgradeneeded = () => req.result.createObjectStore('r', { keyPath: 'key' });
-        for (const [k, v] of Object.entries(old)) if (v && Array.isArray(v.results) && !o[k]) o[k] = v;
+      req.onsuccess = () => resolve(req.result);
-        localStorage.removeItem(LEGACY);
+      req.onerror = req.onblocked = () => resolve(null);
+    } catch { resolve(null); }
+  }));
+  async function tx(mode, fn) {
+    const d = await open();
+    if (!d) return fn(null);
+    return new Promise((resolve) => {
+      let out;
+      try {
+        const t = d.transaction('r', mode);
+        out = fn(t.objectStore('r'));
+        t.oncomplete = () => resolve(out && out.result !== undefined ? out.result : undefined);
+        t.onerror = t.onabort = () => resolve(undefined);
+      } catch { resolve(undefined); }
+    });
   }
-    } catch { /* ignore */ }
+  const idbPut = (rec) => tx('readwrite', (st) => st ? st.put(rec) : void fallback.set(rec.key, rec));
-    mem = o;
+  const idbGet = (key) => tx('readonly', (st) => st ? st.get(key) : { result: fallback.get(key) });
-    return mem;
+  const idbDel = (keys) => tx('readwrite', (st) => { if (st) keys.forEach((k) => st.delete(k)); else keys.forEach((k) => fallback.delete(k)); });
-  }
+  const idbClear = () => tx('readwrite', (st) => st ? st.clear() : void fallback.clear());
-  function write() {
+
-    const o = read();
+  // One-time import of the earlier localStorage stores.
-    for (let attempt = 0; attempt < 4; attempt++) {
+  (async () => {
-      try { localStorage.setItem(KEY, JSON.stringify(o)); return true; } catch {
+    for (const k of OLD_KEYS) {
-        // Quota: drop the oldest fifth and retry.
+      let o = null;
-        const keys = Object.keys(o).sort((a, b) => (o[a].ts || 0) - (o[b].ts || 0));
+      try { o = JSON.parse(localStorage.getItem(k) || 'null'); } catch { o = null; }
-        keys.slice(0, Math.max(1, Math.ceil(keys.length / 5))).forEach((k) => delete o[k]);
+      if (o && typeof o === 'object') {
+        for (const [key, v] of Object.entries(o)) {
+          if (!v || !Array.isArray(v.results) || !v.results.length || index[key]) continue;
+          put(v.query || key, v.results, v.ts);
         }
       }
-    return false;
+      try { localStorage.removeItem(k); } catch { /* blocked */ }
     }
-  return {
+  })();
-    put(q, results) {
+
+  function put(q, results, ts) {
     q = String(q || '').trim();
     if (!q || !Array.isArray(results) || !results.length) return;
-      const o = read();
+    const key = norm(q);
-      o[norm(q)] = { query: q, results: results.slice(0, PER_QUERY), ts: Date.now() };
+    const list = results.slice(0, PER_QUERY);
-      const keys = Object.keys(o);
+    const bytes = JSON.stringify(list).length;
+    index[key] = { query: q, count: list.length, ts: ts || Date.now(), bytes, thumb: (list[0] || {}).thumbnail || '' };
+    idbPut({ key, query: q, results: list, ts: index[key].ts });
+    const keys = Object.keys(index);
     if (keys.length > MAX_QUERIES) {
-        keys.sort((a, b) => (o[a].ts || 0) - (o[b].ts || 0)).slice(0, keys.length - MAX_QUERIES).forEach((k) => delete o[k]);
+      const drop = keys.sort((a, b) => (index[a].ts || 0) - (index[b].ts || 0)).slice(0, keys.length - MAX_QUERIES);
+      drop.forEach((k) => delete index[k]);
+      idbDel(drop);
     }
-      write();
+    persistIndex();
+  }
+  return {
+    FRESH_MS,
+    PER_QUERY,
+    put,
+    // Promise of { query, results, ts } or null.
+    async get(q) {
+      const key = norm(q);
+      if (!index[key]) return null;
+      const rec = await idbGet(key);
+      if (rec && Array.isArray(rec.results) && rec.results.length) return rec;
+      delete index[key]; persistIndex();   // index entry without data
+      return null;
     },
-    get(q) { const h = read()[norm(q)]; return h && Array.isArray(h.results) && h.results.length ? h : null; },
+    has(q) { return !!index[norm(q)]; },
-    has(q) { return !!read()[norm(q)]; },
+    isFresh(ts) { return Date.now() - (ts || 0) < FRESH_MS; },
-    // Newest first: [{ query, count, ts, thumb }]
+    // Newest first: [{ key, query, count, ts, thumb }]
     list() {
-      const o = read();
+      return Object.keys(index).map((k) => ({ key: k, query: index[k].query || k, count: index[k].count || 0, ts: index[k].ts || 0, thumb: index[k].thumb || '' }))
-      return Object.keys(o).map((k) => ({ key: k, query: o[k].query || k, count: (o[k].results || []).length, ts: o[k].ts || 0, thumb: ((o[k].results || [])[0] || {}).thumbnail || '' }))
         .sort((a, b) => b.ts - a.ts);
     },
-    remove(keys) { const o = read(); (keys || []).forEach((k) => delete o[k]); write(); },
+    remove(keys) { (keys || []).forEach((k) => delete index[k]); idbDel(keys || []); persistIndex(); },
-    clear() { mem = {}; try { localStorage.removeItem(KEY); } catch { /* blocked */ } },
+    clear() { index = {}; idbClear(); persistIndex(); },
-    bytes() { try { return (localStorage.getItem(KEY) || '').length; } catch { return 0; } },
+    bytes() { return Object.values(index).reduce((n, x) => n + (x.bytes || 0), 0); },
-    count() { return Object.keys(read()).length; },
+    count() { return Object.keys(index).length; },
   };
 })();
 
@@ -343,9 +389,13 @@ async function opfsClear() {
 let _currentBlobUrl = null;
 
 const API = {
-  search: (query) => WEB
+  search: (query, { refresh = false } = {}) => WEB
-    ? webFetch(`/api/search?q=${encodeURIComponent(query)}`)
+    ? webFetch(`/api/search?q=${encodeURIComponent(query)}${refresh ? '&refresh=1' : ''}`)
     : call('yt.search', 'yt_search', { query }),
+  // Videos the server already knows — answers fast while the real search loads.
+  searchLocal: (query) => WEB
+    ? webFetch(`/api/search/local?q=${encodeURIComponent(query)}`)
+    : Promise.resolve({ ok: false }),
   getChannel: (channel) => WEB
     ? webFetch(`/api/channel?c=${encodeURIComponent(channel)}`)
     : call('yt.channel', 'yt_channel', { channel }),
@@ -1303,7 +1353,7 @@ function startCacheResyncWatch() {
 // Pull a result set's artwork into the service worker's thumbnail cache, a few at
 // a time, so a saved search still has pictures offline.
 function warmSearchThumbs(results) {
-  const urls = (results || []).map((r) => r && r.thumbnail).filter(Boolean).slice(0, 50);
+  const urls = (results || []).map((r) => r && r.thumbnail).filter(Boolean).slice(0, 100);
   let i = 0;
   const next = () => { if (i >= urls.length) return; warmThumb(urls[i++]); setTimeout(next, 40); };
   next(); next(); next();
@@ -7476,6 +7526,7 @@ function setListPaneHidden(hidden) {
 
 function renderList() {
   { const ls = $('listSub'); if (ls) ls.textContent = ''; const ps = $('p2pStrip'); if (ps) ps.classList.add('hidden'); }
+  if (view.type !== 'search') { const sm = $('searchMeta'); if (sm) sm.classList.add('hidden'); }
   setListPaneHidden(view.type === 'search' && !searchResults.length && !searchAttempted);
   if (view.type === 'settings') { renderSettings(); return; }
   if (view.type === 'saved') { renderSaved(); return; }
@@ -7878,7 +7929,7 @@ function renderSavedSearches() {
     row.querySelector('.card-title').textContent = it.query;
     row.addEventListener('click', () => {
       els.searchInput.value = it.query;
-      if (els.runSearch) els.runSearch(it.query, { instant: SearchLibrary.get(it.query) });
+      if (els.runSearch) els.runSearch(it.query);
     });
     c.appendChild(row);
   });
@@ -9535,7 +9586,11 @@ function wireUI() {
   // `searchSeq` guards against a slower response overwriting a newer one
   // (the instant paint above can itself be "in flight" behind a background
   // refresh when the user searches something else right after).
-  async function runSearchQuery(q, { instant = null } = {}) {
+  // The saved copy of `q` is read from IndexedDB. A copy younger than 15 days is
+  // shown as is (no network); an older one is shown at once and refreshed behind
+  // it. `refresh` (the "Update search" button) always asks the server for fresh
+  // results, bypassing its caches too.
+  async function runSearchQuery(q, { refresh = false } = {}) {
     q = (q || '').trim();
     if (!q) return;
     const mySeq = ++searchSeq;
@@ -9543,10 +9598,16 @@ function wireUI() {
     RecentSearches.add(q);
     hideRecentSearches();
     view = { type: 'search' };
+    const instant = refresh ? null : await RecentSearches.getResults(q);
+    if (mySeq !== searchSeq) return;
     if (instant) {
       searchResults = instant.results;
       searchAttempted = true;
       render();
+      showSearchMeta(q, instant.ts);
+      // Even on a fresh copy, make sure thumbnails are in the SW cache.
+      warmSearchThumbs(instant.results);
+      if (SearchLibrary.isFresh(instant.ts)) { scrollListIntoViewPortrait(); return; }
     } else {
       render();
       showSearchSkeletons();
@@ -9554,27 +9615,60 @@ function wireUI() {
     // Even when already on the search view (no view change for render() to
     // detect), a new query means the user wants to see the results.
     scrollListIntoViewPortrait();
+    const metaBtn = $('searchUpdateBtn');
+    if (metaBtn) { metaBtn.disabled = true; metaBtn.textContent = 'Updating…'; }
+    let networkDone = false, paintedLocal = false;
+    if (!instant && !refresh) {
+      // Known videos from the server's metadata store, painted until YouTube answers.
+      API.searchLocal(q).then((lr) => {
+        if (networkDone || mySeq !== searchSeq || !lr || !lr.ok || !(lr.results || []).length) return;
+        paintedLocal = true;
+        searchResults = lr.results;
+        searchAttempted = true;
+        renderList();
+      }).catch(() => {});
+    }
     try {
-      const res = await API.search(q);
+      const res = await API.search(q, { refresh });
+      networkDone = true;
       if (mySeq !== searchSeq) return; // superseded by a newer search
       if (!res || !res.ok) throw new Error(res?.error || 'Search failed');
       const results = res.results || [];
       searchResults = results;
       searchAttempted = true;
       renderList();
-      RecentSearches.cacheResults(q, results);
+      const ts = Date.now();
+      RecentSearches.cacheResults(q, results, ts);
+      showSearchMeta(q, ts);
       warmSearchThumbs(results);
       warmStreams(results.slice(0, 3).map((r) => r.id));
+      if (refresh) toast('Search updated');
       perfMeasure('ytp:search', 'ytp:search-start');
     } catch (err) {
+      networkDone = true;
       if (mySeq !== searchSeq) return;
-      if (instant) return; // keep showing the cached results; fail silently
+      if (paintedLocal) return; // keep the server's known videos
+      if (instant) { showSearchMeta(q, instant.ts); if (refresh) toast('Could not update: ' + err.message); return; } // keep the saved copy
       els.cards.innerHTML = '';
       els.status.classList.remove('hidden');
       els.status.textContent = '⚠ ' + err.message;
     }
   }
 
+  // "Saved copy · 3 d ago  [↻ Update search]" above the results.
+  function showSearchMeta(q, ts) {
+    const m = $('searchMeta');
+    if (!m || view.type !== 'search' || !searchResults.length) return;
+    const left = Math.max(0, Math.ceil((SearchLibrary.FRESH_MS - (Date.now() - ts)) / 86400000));
+    m.innerHTML = `<span class="sm-info">Updated ${fmtAgo(ts)} · ${searchResults.length} results${SearchLibrary.isFresh(ts) ? ` · kept ${left} more day${left === 1 ? '' : 's'}` : ' · refreshing'}</span>`
+      + '<button type="button" id="searchUpdateBtn" class="sm-btn">↻ Update search</button>';
+    m.classList.remove('hidden');
+    m.dataset.q = q;
+  }
+  $('searchMeta').addEventListener('click', (e) => {
+    if (e.target.closest('#searchUpdateBtn')) runSearchQuery($('searchMeta').dataset.q || '', { refresh: true });
+  });
+
   els.runSearch = runSearchQuery;
   els.searchForm.addEventListener('submit', (e) => {
     e.preventDefault();
@@ -9635,7 +9729,7 @@ function wireUI() {
       e.preventDefault();
       const q = row.dataset.q;
       els.searchInput.value = q;
-      runSearchQuery(q, { instant: RecentSearches.getResults(q) });
+      runSearchQuery(q);
     }
   });
   document.addEventListener('click', (e) => {

frontend/index.html

@@ -373,6 +373,7 @@
               <button id="batchDeleteBtn" class="batch-btn batch-danger">Delete</button>
               <button id="batchCancelBtn" class="batch-btn">Cancel</button>
             </div>
+            <div id="searchMeta" class="search-meta hidden"></div>
             <div id="status" class="status hidden"></div>
             <div id="cards" class="cards"></div>
             <div id="p2pStrip" class="p2p-strip hidden"></div>

frontend/refresh.css

@@ -545,3 +545,9 @@
 .ss-row small { color: var(--text-dim); flex: none; font-size: 11px; }
 .search-recent-clear small { opacity: 0.6; }
 .topbar { position: relative; z-index: 80; } /* keeps the recent-searches dropdown above the page content */
+
+/* Search results: age line + "Update search" */
+.search-meta { display: flex; align-items: center; justify-content: space-between; gap: 10px; margin: 0 0 10px; padding: 8px 12px; border: 1px solid var(--line); border-radius: 12px; font-size: 12px; color: var(--text-dim); }
+.search-meta.hidden { display: none; }
+.search-meta .sm-btn { flex: none; border: 1px solid var(--line); background: transparent; color: var(--text); border-radius: 999px; padding: 6px 12px; font: inherit; font-weight: 600; cursor: pointer; }
+.search-meta .sm-btn:disabled { opacity: .55; cursor: default; }

server/db.js

@@ -123,6 +123,17 @@ export async function initDb() {
     );
     CREATE INDEX IF NOT EXISTS idx_search_cache_lru ON search_cache (last_access);
 
+    -- Every video the server has ever seen in a search: lets a search show
+    -- known videos at once while YouTube is still loading.
+    CREATE TABLE IF NOT EXISTS video_meta (
+      id         TEXT    PRIMARY KEY,
+      card       TEXT    NOT NULL,            -- the result card, JSON
+      hay        TEXT    NOT NULL,            -- lowercased title + channel, for matching
+      seen       INTEGER NOT NULL DEFAULT 1,  -- times it appeared in a search
+      updated_at INTEGER NOT NULL DEFAULT 0
+    );
+    CREATE INDEX IF NOT EXISTS idx_video_meta_upd ON video_meta (updated_at);
+
     -- Shared per-video documents (kind = lyrics | chapters), visible to every
     -- user. The live copy is here; every save also lands in video_note_revs
     -- as a full snapshot, which is the server-side backup and undo history.

server/search-cache.js

@@ -58,3 +58,38 @@ export async function stats() {
   const t = (await db.execute('SELECT COUNT(*) AS n, COALESCE(SUM(size),0) AS b FROM search_cache')).rows[0];
   return { queries: Number(t.n), bytes: Number(t.b), maxQueries: MAX_ROWS, maxBytes: MAX_BYTES };
 }
+
+// ---- Known-video metadata: instant results while YouTube loads ----
+export const MAX_VIDEOS = Number(process.env.VIDEO_META_MAX) || 500_000;
+
+export async function rememberVideos(cards) {
+  const now = Date.now();
+  const rows = (cards || []).filter((c) => c && typeof c.id === 'string' && c.title);
+  if (!rows.length) return;
+  await db.batch(rows.map((c) => ({
+    sql: `INSERT INTO video_meta (id, card, hay, seen, updated_at) VALUES (?,?,?,1,?)
+          ON CONFLICT(id) DO UPDATE SET card=excluded.card, hay=excluded.hay, seen=seen+1, updated_at=excluded.updated_at`,
+    args: [c.id, JSON.stringify(c), `${c.title} ${c.channel || ''}`.toLowerCase(), now],
+  })));
+  if (Math.random() < 0.02) trimVideos().catch(() => {});
+}
+
+// Cards whose title/channel contain EVERY word of the query, most-seen first.
+export async function searchVideos(q, limit = 60) {
+  const words = String(q).toLowerCase().split(/\s+/).filter(Boolean).slice(0, 8);
+  if (!words.length) return [];
+  const like = (w) => '%' + w.replace(/[\\%_]/g, '\\$&') + '%';
+  const r = await db.execute({
+    sql: `SELECT card FROM video_meta WHERE ${words.map(() => "hay LIKE ? ESCAPE '\\'").join(' AND ')}
+          ORDER BY seen DESC, updated_at DESC LIMIT ?`,
+    args: [...words.map(like), limit],
+  });
+  return r.rows.map((x) => { try { return JSON.parse(x.card); } catch { return null; } }).filter(Boolean);
+}
+
+export async function trimVideos(max = MAX_VIDEOS) {
+  const n = Number((await db.execute('SELECT COUNT(*) AS n FROM video_meta')).rows[0].n);
+  if (n <= max) return 0;
+  await db.execute({ sql: 'DELETE FROM video_meta WHERE id IN (SELECT id FROM video_meta ORDER BY updated_at ASC LIMIT ?)', args: [n - max] });
+  return n - max;
+}

server/search-cache.test.js

@@ -18,3 +18,14 @@ test('trim evicts least-recently-used rows over the row cap', async () => {
   expect(dropped).toBeGreaterThan(0);
   expect((await sc.stats()).queries).toBe(3);
 });
+
+test('known videos are remembered and found by every word', async () => {
+  await sc.rememberVideos([
+    { id: 'a1', title: 'Way Maker - Sinach', channel: 'Worship' },
+    { id: 'b2', title: 'Way Maker (live)', channel: 'Other' },
+    { id: 'c3', title: 'Goodness of God', channel: 'Worship' },
+  ]);
+  expect((await sc.searchVideos('way maker')).map((c) => c.id).sort()).toEqual(['a1', 'b2']);
+  expect((await sc.searchVideos('maker worship')).map((c) => c.id)).toEqual(['a1']);
+  expect(await sc.searchVideos('100%')).toEqual([]);
+});

server/server.js

@@ -428,6 +428,7 @@ function fetchYoutube(q) {
     const fetchedAt = Date.now();
     if (searchCache.size >= SEARCH_CACHE_MAX) searchCache.delete(searchCache.keys().next().value);
     searchCache.set(key, { yt, fetchedAt });
+    if (yt.length) searchCacheDb.rememberVideos(yt).catch(() => {});
     if (yt.length) searchCacheDb.put(q, yt).catch((e) => console.warn(`[search] cache write: ${e.message}`));
     return { yt, fetchedAt };
   })().finally(() => inflightSearches.delete(key));
@@ -440,6 +441,16 @@ const searchLibrary = async (q) => {
   try { return (await notesDb.listUploads({ q, limit: 20 })).map(uploads.card); } catch { return []; }
 };
 
+// GET /api/search/local?q=<query> — videos this server already knows (from every
+// earlier search) plus its own uploads; answers in milliseconds so the app can
+// show something while the real search is still loading.
+app.get('/api/search/local', async (c) => {
+  const q = (c.req.query('q') || '').trim();
+  if (!q) return c.json({ ok: false, error: 'empty query' }, 400);
+  const [mine, known] = await Promise.all([searchLibrary(q), searchCacheDb.searchVideos(q).catch(() => [])]);
+  return c.json({ ok: true, results: [...mine, ...known], local: true });
+});
+
 // GET /api/search?q=<query>[&refresh=1]
 // Memory → persistent cache (fresh 10 min, then stale-while-revalidate up to 15
 // days: answered instantly, refreshed in the background) → YouTube. `refresh=1`