From 5c94a7d46a62d671f1d480c0147f9d7541a186ea Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Thu, 15 Oct 2026 10:30:20 +0800 Subject: [PATCH] Search: answer on the first YouTube page, fill the rest in background; never cache empty results; gzip search JSON --- frontend/app.js | 24 ++++++++++++++++++++---- server/server.js | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/frontend/app.js b/frontend/app.js index 0b093b8..32c2ce3 100755 --- a/frontend/app.js +++ b/frontend/app.js @@ -64,6 +64,7 @@ async function webFetch(path, opts = {}) { return AsyncGuard.getJson(path, opts, { // Some upstream failures are represented by HTTP 200 + {ok:false}; do // not expose those transient responses as a manual Retry state either. + ...(/^\/api\/search\?/.test(path) ? { timeoutMs: 40000, attempts: 2 } : {}), retryResult: value => value?.ok === false && /\b(?:408|425|429|5\d\d)\b|temporar|timed?\s*out|network|connection|upstream|unavailable|available right now/i.test(String(value.error || value.message || '')), }); const res = await fetch(path, opts); @@ -170,10 +171,12 @@ const RecentSearches = (() => { }, clear() { save([]); }, // saved results stay in SearchLibrary (cleared only from Settings) // Cache the just-fetched result set for `q` (called on every successful search). - cacheResults(q, results, ts) { SearchLibrary.put(q, results, ts); }, - // Promise of the cached {query, results, ts} for `q`, or null. - getResults(q) { - return SearchLibrary.get(q); + cacheResults(q, results, ts) { if (results && results.length) SearchLibrary.put(q, results, ts); }, + // Promise of the cached {query, results, ts} for `q`, or null. An empty + // saved set (left by an earlier failed search) counts as no copy. + async getResults(q) { + const hit = await SearchLibrary.get(q); + return hit && hit.results && hit.results.length ? hit : null; }, // Normalized queries that have a cached result set — one localStorage // read for the whole dropdown, rather than one per row. @@ -7893,6 +7896,19 @@ function wireUI() { warmStreams(results.slice(0, 3).map((r) => r.id)); if (refresh) toast('Search updated'); perfMeasure('ytp:search', 'ytp:search-start'); + // The server answered with YouTube's first page and is still fetching + // the rest; pick up the full list quietly once it has landed. + if (res.partial) setTimeout(async () => { + if (mySeq !== searchSeq || view.type !== 'search') return; + try { + const more = await API.search(q); + if (mySeq !== searchSeq || !more?.ok || (more.results || []).length <= results.length) return; + searchResults = more.results; + renderList(); + RecentSearches.cacheResults(q, more.results, Date.now()); + showSearchMeta(q, Date.now()); + } catch { /* keep the first page */ } + }, 10000); } catch (err) { networkDone = true; if (mySeq !== searchSeq) return; diff --git a/server/server.js b/server/server.js index 9a402fe..bcc27b3 100644 --- a/server/server.js +++ b/server/server.js @@ -383,14 +383,22 @@ const inflightSearches = new Map(); // lowercased q -> Promise<{ yt, youtubeErro // The YouTube side of one search (up to 200 cards), shared by every caller asking // the same thing at the same time, and written to memory + the persistent cache. +// Resolves as soon as the first InnerTube page arrives (~20 cards, under a +// second); the remaining pages keep loading in the background and land in the +// caches, so the next ask for the same query gets the full list. Paging all +// 200 cards before answering took 7-12 s and timed out on slow phone links. function fetchYoutube(q) { const key = q.toLowerCase(); if (inflightSearches.has(key)) return inflightSearches.get(key); - const p = (async () => { + let answered = false, resolveEarly; + const early = new Promise((r) => { resolveEarly = r; }); + const full = (async () => { let yt = []; if (process.env.SEARCH_INNERTUBE !== '0') { try { - yt = await innertube.searchDeep(q, { limit: searchCacheDb.PER_QUERY }); + yt = await innertube.searchDeep(q, { limit: searchCacheDb.PER_QUERY, onPage: (cards) => { + if (!answered && cards.length) { answered = true; resolveEarly({ yt: cards, fetchedAt: Date.now(), partial: true }); } + } }); } catch (e) { console.warn(`[search] innertube failed, using yt-dlp: ${e.message}`); } @@ -404,12 +412,17 @@ function fetchYoutube(q) { yt = parseCards(out); } const fetchedAt = Date.now(); - if (searchCache.size >= SEARCH_CACHE_MAX) searchCache.delete(searchCache.keys().next().value); - searchCache.set(key, { yt, fetchedAt }); - if (yt.length) await searchCacheDb.rememberVideos(yt); - if (yt.length) searchCacheDb.put(q, yt).catch((e) => console.warn(`[search] cache write: ${e.message}`)); + // An empty answer is a transient failure, never a result worth caching. + if (yt.length) { + if (searchCache.size >= SEARCH_CACHE_MAX) searchCache.delete(searchCache.keys().next().value); + searchCache.set(key, { yt, fetchedAt }); + await searchCacheDb.rememberVideos(yt); + searchCacheDb.put(q, yt).catch((e) => console.warn(`[search] cache write: ${e.message}`)); + } return { yt, fetchedAt }; })().finally(() => inflightSearches.delete(key)); + full.catch((e) => { if (answered) console.warn(`[search] background fill: ${e.message}`); }); + const p = Promise.race([early, full]); inflightSearches.set(key, p); return p; } @@ -439,7 +452,13 @@ app.get('/api/search', async (c) => { if (!q) return c.json({ ok: false, error: 'empty query' }, 400); const key = q.toLowerCase(); const mine = await searchLibrary(q); - const answer = (yt, extra = {}) => c.json({ ok: true, results: [...mine, ...yt], ...extra }); + const answer = (yt, extra = {}) => { + const body = JSON.stringify({ ok: true, results: [...mine, ...yt], ...extra }); + const headers = { 'Content-Type': 'application/json; charset=utf-8', Vary: 'Accept-Encoding' }; + if (/\bgzip\b/.test(c.req.header('accept-encoding') || '') && body.length > 1024) + return new Response(Bun.gzipSync(body), { headers: { ...headers, 'Content-Encoding': 'gzip' } }); + return new Response(body, { headers }); + }; if (c.req.query('refresh') !== '1') { let hit = searchCache.get(key); if (!hit) { @@ -454,8 +473,9 @@ app.get('/api/search', async (c) => { } } try { - const { yt, fetchedAt } = await fetchYoutube(q); - return answer(yt, { fetchedAt }); + const { yt, fetchedAt, partial } = await fetchYoutube(q); + if (!yt.length && !mine.length) return c.json({ ok: false, error: 'YouTube returned no results right now, try again' }, 503); + return answer(yt, partial ? { fetchedAt, partial: true } : { fetchedAt }); } catch (err) { if (mine.length) return c.json({ ok: true, results: mine, youtubeError: err.message }); return c.json({ ok: false, error: err.message }, 500);