Search: answer on the first YouTube page, fill the rest in background; never cache empty results; gzip search JSON

frontend/app.js

@@ -64,6 +64,7 @@ async function webFetch(path, opts = {}) {
     return AsyncGuard.getJson(path, opts, {
       // Some upstream failures are represented by HTTP 200 + {ok:false}; do
       // not expose those transient responses as a manual Retry state either.
+      ...(/^\/api\/search\?/.test(path) ? { timeoutMs: 40000, attempts: 2 } : {}),
       retryResult: value => value?.ok === false && /\b(?:408|425|429|5\d\d)\b|temporar|timed?\s*out|network|connection|upstream|unavailable|available right now/i.test(String(value.error || value.message || '')),
     });
   const res = await fetch(path, opts);
@@ -170,10 +171,12 @@ const RecentSearches = (() => {
     },
     clear() { save([]); }, // saved results stay in SearchLibrary (cleared only from Settings)
     // Cache the just-fetched result set for `q` (called on every successful search).
-    cacheResults(q, results, ts) { SearchLibrary.put(q, results, ts); },
+    cacheResults(q, results, ts) { if (results && results.length) SearchLibrary.put(q, results, ts); },
-    // Promise of the cached {query, results, ts} for `q`, or null.
+    // Promise of the cached {query, results, ts} for `q`, or null. An empty
-    getResults(q) {
+    // saved set (left by an earlier failed search) counts as no copy.
-      return SearchLibrary.get(q);
+    async getResults(q) {
+      const hit = await SearchLibrary.get(q);
+      return hit && hit.results && hit.results.length ? hit : null;
     },
     // Normalized queries that have a cached result set — one localStorage
     // read for the whole dropdown, rather than one per row.
@@ -7893,6 +7896,19 @@ function wireUI() {
       warmStreams(results.slice(0, 3).map((r) => r.id));
       if (refresh) toast('Search updated');
       perfMeasure('ytp:search', 'ytp:search-start');
+      // The server answered with YouTube's first page and is still fetching
+      // the rest; pick up the full list quietly once it has landed.
+      if (res.partial) setTimeout(async () => {
+        if (mySeq !== searchSeq || view.type !== 'search') return;
+        try {
+          const more = await API.search(q);
+          if (mySeq !== searchSeq || !more?.ok || (more.results || []).length <= results.length) return;
+          searchResults = more.results;
+          renderList();
+          RecentSearches.cacheResults(q, more.results, Date.now());
+          showSearchMeta(q, Date.now());
+        } catch { /* keep the first page */ }
+      }, 10000);
     } catch (err) {
       networkDone = true;
       if (mySeq !== searchSeq) return;

server/server.js

@@ -383,14 +383,22 @@ const inflightSearches = new Map(); // lowercased q -> Promise<{ yt, youtubeErro
 
 // The YouTube side of one search (up to 200 cards), shared by every caller asking
 // the same thing at the same time, and written to memory + the persistent cache.
+// Resolves as soon as the first InnerTube page arrives (~20 cards, under a
+// second); the remaining pages keep loading in the background and land in the
+// caches, so the next ask for the same query gets the full list. Paging all
+// 200 cards before answering took 7-12 s and timed out on slow phone links.
 function fetchYoutube(q) {
   const key = q.toLowerCase();
   if (inflightSearches.has(key)) return inflightSearches.get(key);
-  const p = (async () => {
+  let answered = false, resolveEarly;
+  const early = new Promise((r) => { resolveEarly = r; });
+  const full = (async () => {
     let yt = [];
     if (process.env.SEARCH_INNERTUBE !== '0') {
       try {
-        yt = await innertube.searchDeep(q, { limit: searchCacheDb.PER_QUERY });
+        yt = await innertube.searchDeep(q, { limit: searchCacheDb.PER_QUERY, onPage: (cards) => {
+          if (!answered && cards.length) { answered = true; resolveEarly({ yt: cards, fetchedAt: Date.now(), partial: true }); }
+        } });
       } catch (e) {
         console.warn(`[search] innertube failed, using yt-dlp: ${e.message}`);
       }
@@ -404,12 +412,17 @@ function fetchYoutube(q) {
       yt = parseCards(out);
     }
     const fetchedAt = Date.now();
-    if (searchCache.size >= SEARCH_CACHE_MAX) searchCache.delete(searchCache.keys().next().value);
+    // An empty answer is a transient failure, never a result worth caching.
-    searchCache.set(key, { yt, fetchedAt });
+    if (yt.length) {
-    if (yt.length) await searchCacheDb.rememberVideos(yt);
+      if (searchCache.size >= SEARCH_CACHE_MAX) searchCache.delete(searchCache.keys().next().value);
-    if (yt.length) searchCacheDb.put(q, yt).catch((e) => console.warn(`[search] cache write: ${e.message}`));
+      searchCache.set(key, { yt, fetchedAt });
+      await searchCacheDb.rememberVideos(yt);
+      searchCacheDb.put(q, yt).catch((e) => console.warn(`[search] cache write: ${e.message}`));
+    }
     return { yt, fetchedAt };
   })().finally(() => inflightSearches.delete(key));
+  full.catch((e) => { if (answered) console.warn(`[search] background fill: ${e.message}`); });
+  const p = Promise.race([early, full]);
   inflightSearches.set(key, p);
   return p;
 }
@@ -439,7 +452,13 @@ app.get('/api/search', async (c) => {
   if (!q) return c.json({ ok: false, error: 'empty query' }, 400);
   const key = q.toLowerCase();
   const mine = await searchLibrary(q);
-  const answer = (yt, extra = {}) => c.json({ ok: true, results: [...mine, ...yt], ...extra });
+  const answer = (yt, extra = {}) => {
+    const body = JSON.stringify({ ok: true, results: [...mine, ...yt], ...extra });
+    const headers = { 'Content-Type': 'application/json; charset=utf-8', Vary: 'Accept-Encoding' };
+    if (/\bgzip\b/.test(c.req.header('accept-encoding') || '') && body.length > 1024)
+      return new Response(Bun.gzipSync(body), { headers: { ...headers, 'Content-Encoding': 'gzip' } });
+    return new Response(body, { headers });
+  };
   if (c.req.query('refresh') !== '1') {
     let hit = searchCache.get(key);
     if (!hit) {
@@ -454,8 +473,9 @@ app.get('/api/search', async (c) => {
     }
   }
   try {
-    const { yt, fetchedAt } = await fetchYoutube(q);
+    const { yt, fetchedAt, partial } = await fetchYoutube(q);
-    return answer(yt, { fetchedAt });
+    if (!yt.length && !mine.length) return c.json({ ok: false, error: 'YouTube returned no results right now, try again' }, 503);
+    return answer(yt, partial ? { fetchedAt, partial: true } : { fetchedAt });
   } catch (err) {
     if (mine.length) return c.json({ ok: true, results: mine, youtubeError: err.message });
     return c.json({ ok: false, error: err.message }, 500);