Persistent server search cache: 200 results per query, 250k queries, 15-day stale-while-revalidate, refresh=1

server/innertube.js

@@ -8,10 +8,8 @@ export function lengthToSeconds(s) {
   return s.trim().split(':').map(Number).reduce((acc, n) => acc * 60 + n, 0);
 }
 
-export function parseSearch(json) {
-  const sections = json?.contents?.twoColumnSearchResultsRenderer?.primaryContents
-    ?.sectionListRenderer?.contents;
-  if (!Array.isArray(sections)) throw new Error('innertube: unexpected response shape');
+// Cards from a list of itemSectionRenderer-style sections.
+function cardsFromSections(sections) {
   const out = [];
   for (const s of sections) {
     for (const it of s?.itemSectionRenderer?.contents || []) {
@@ -34,6 +32,37 @@ export function parseSearch(json) {
   return out;
 }
 
+const tokenOf = (sections) => {
+  for (const s of sections || []) {
+    const t = s?.continuationItemRenderer?.continuationEndpoint?.continuationCommand?.token;
+    if (t) return t;
+  }
+  return '';
+};
+
+export function parseSearch(json) {
+  const sections = json?.contents?.twoColumnSearchResultsRenderer?.primaryContents
+    ?.sectionListRenderer?.contents;
+  if (!Array.isArray(sections)) throw new Error('innertube: unexpected response shape');
+  return cardsFromSections(sections);
+}
+
+// First page plus the token for the next one ('' when there is none).
+export function parseSearchPage(json) {
+  const sections = json?.contents?.twoColumnSearchResultsRenderer?.primaryContents
+    ?.sectionListRenderer?.contents;
+  if (!Array.isArray(sections)) throw new Error('innertube: unexpected response shape');
+  return { cards: cardsFromSections(sections), next: tokenOf(sections) };
+}
+
+// A continuation response: more cards plus the next token.
+export function parseContinuation(json) {
+  const items = (json?.onResponseReceivedCommands || [])
+    .flatMap((c) => c?.appendContinuationItemsAction?.continuationItems || []);
+  if (!Array.isArray(items) || !items.length) return { cards: [], next: '' };
+  return { cards: cardsFromSections(items), next: tokenOf(items) };
+}
+
 export async function search(q, { fetchImpl = fetch, timeoutMs = 6000 } = {}) {
   const res = await fetchImpl('https://www.youtube.com/youtubei/v1/search?prettyPrint=false', {
     method: 'POST',
@@ -44,3 +73,34 @@ export async function search(q, { fetchImpl = fetch, timeoutMs = 6000 } = {}) {
   if (!res.ok) throw new Error(`innertube: HTTP ${res.status}`);
   return parseSearch(await res.json());
 }
+
+// Up to `limit` results (default 200) by following continuation tokens. The pages
+// are a chain, so they are fetched one after another; every page is optional —
+// a failure part-way returns what was gathered. `onPage(cards)` lets a caller
+// save progress.
+export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs = 6000, maxPages = 14, onPage } = {}) {
+  const post = async (body) => {
+    const res = await fetchImpl('https://www.youtube.com/youtubei/v1/search?prettyPrint=false', {
+      method: 'POST', headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ context: { client: CLIENT }, ...body }), signal: AbortSignal.timeout(timeoutMs),
+    });
+    if (!res.ok) throw new Error(`innertube: HTTP ${res.status}`);
+    return res.json();
+  };
+  const first = parseSearchPage(await post({ query: q }));
+  const seen = new Set();
+  const all = [];
+  const add = (cards) => { for (const c of cards) if (!seen.has(c.id)) { seen.add(c.id); all.push(c); } };
+  add(first.cards);
+  if (onPage) onPage(all.slice());
+  let next = first.next;
+  for (let i = 0; i < maxPages && next && all.length < limit; i++) {
+    let page;
+    try { page = parseContinuation(await post({ continuation: next })); } catch { break; }
+    if (!page.cards.length) break;
+    add(page.cards);
+    if (onPage) onPage(all.slice());
+    next = page.next;
+  }
+  return all.slice(0, limit);
+}