Recover related videos with bounded discovery and retryable fallbacks

server/innertube.js

@@ -104,3 +104,42 @@ export async function searchDeep(q, { limit = 200, fetchImpl = fetch, timeoutMs
   }
   return all.slice(0, limit);
 }
+
+// Watch-next recommendations use compact renderers or the newer lockup model.
+// Traverse only the recommendation contents, never player configuration/ads.
+export function parseRelated(json) {
+  const contents = json?.contents?.twoColumnWatchNextResults?.secondaryResults?.secondaryResults?.results
+    || json?.contents?.singleColumnWatchNextResults?.results?.results?.contents;
+  if (!Array.isArray(contents)) throw new Error('innertube: unexpected related response shape');
+  const cards = [], seen = new Set();
+  function visit(value) {
+    if (!value || typeof value !== 'object') return;
+    const renderer = value.compactVideoRenderer || value.videoRenderer;
+    const model = value.lockupViewModel;
+    const id = renderer?.videoId || (model?.contentType === 'LOCKUP_CONTENT_TYPE_VIDEO' ? model.contentId : '');
+    if (typeof id === 'string' && /^[\w-]{11}$/.test(id) && !seen.has(id)) {
+      const meta = model?.metadata?.lockupMetadataViewModel;
+      const part = meta?.metadata?.contentMetadataViewModel?.metadataRows?.[0]?.metadataParts?.[0]?.text;
+      const owner = renderer?.shortBylineText?.runs?.[0] || renderer?.longBylineText?.runs?.[0] || {};
+      const browse = owner.navigationEndpoint?.browseEndpoint || part?.commandRuns?.[0]?.onTap?.innertubeCommand?.browseEndpoint || {};
+      const badge = model?.contentImage?.thumbnailViewModel?.overlays?.flatMap(o => o.thumbnailOverlayBadgeViewModel?.thumbnailBadges || []).find(b => b.thumbnailBadgeViewModel?.text)?.thumbnailBadgeViewModel?.text;
+      const title = renderer?.title?.simpleText || renderer?.title?.runs?.map(r => r.text).join('') || meta?.title?.content;
+      if (title) { seen.add(id); cards.push({ id, title, channel: owner.text || part?.content || '', channelId: browse.browseId || '', channelUrl: browse.canonicalBaseUrl ? `https://www.youtube.com${browse.canonicalBaseUrl}` : '', duration: lengthToSeconds(renderer?.lengthText?.simpleText || badge), thumbnail: `https://i.ytimg.com/vi/${id}/mqdefault.jpg` }); }
+      return;
+    }
+    // Ignore advertised placements rather than recursing into their videos.
+    for (const [key, child] of Object.entries(value)) if (!/adSlot|promoted|adPlacement/i.test(key)) {
+      if (Array.isArray(child)) child.forEach(visit); else visit(child);
+    }
+  }
+  contents.forEach(visit);
+  return cards;
+}
+export async function related(videoId, { fetchImpl = fetch, timeoutMs = 4000 } = {}) {
+  const response = await fetchImpl('https://www.youtube.com/youtubei/v1/next?prettyPrint=false', {
+    method: 'POST', headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ context: { client: CLIENT }, videoId }), signal: AbortSignal.timeout(timeoutMs),
+  });
+  if (!response.ok) throw new Error(`innertube related: HTTP ${response.status}`);
+  return parseRelated(await response.json());
+}

server/innertube.test.js

@@ -47,3 +47,16 @@ test('searchDeep follows continuation tokens, de-duplicates and stops at the lim
 test('parseContinuation of an empty response is empty', () => {
   expect(parseContinuation({})).toEqual({ cards: [], next: '' });
 });
+
+test('related parser reads compact and modern lockup cards, excluding playlists and ads', async () => {
+  const { parseRelated } = await import('./innertube.js');
+  const compact = { compactVideoRenderer: { videoId: 'aaaaaaaaaaa', title: { simpleText: 'First' }, shortBylineText: { runs: [{ text: 'Artist' }] }, lengthText: { simpleText: '3:12' } } };
+  const modern = { lockupViewModel: { contentId: 'bbbbbbbbbbb', contentType: 'LOCKUP_CONTENT_TYPE_VIDEO', metadata: { lockupMetadataViewModel: { title: { content: 'Second' }, metadata: { contentMetadataViewModel: { metadataRows: [{ metadataParts: [{ text: { content: 'Band' } }] }] } } } } } };
+  const result = parseRelated({ contents: { twoColumnWatchNextResults: { secondaryResults: { secondaryResults: { results: [compact, modern, compact, { lockupViewModel: { ...modern.lockupViewModel, contentType: 'LOCKUP_CONTENT_TYPE_PLAYLIST' } }, { adSlotRenderer: { contents: compact } }] } } } } });
+  expect(result.map(c => [c.id, c.title, c.channel])).toEqual([['aaaaaaaaaaa', 'First', 'Artist'], ['bbbbbbbbbbb', 'Second', 'Band']]);expect(result[0].duration).toBe(192);
+});
+test('related HTTP failures and unsupported shapes throw for search fallback', async () => {
+  const { related, parseRelated } = await import('./innertube.js');
+  expect(() => parseRelated({ error: {} })).toThrow('response shape');
+  await expect(related('aaaaaaaaaaa', { fetchImpl: async () => ({ ok: false, status: 429 }) })).rejects.toThrow('429');
+});

server/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "start": "bun server.js",
     "dev": "bun --hot server.js",
-    "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./ytdlp-pool.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js"
+    "test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js"
   },
   "dependencies": {
     "@hono/node-server": "^1.14.0",

server/related.js

@@ -0,0 +1,65 @@
+// Related discovery is bounded and independent of the deep search collector.
+export function relatedQueries({ title = '', channel = '' }) {
+  title = String(title).replace(/\([^)]*(?:official|lyrics?|music video)[^)]*\)|\[[^\]]*(?:official|lyrics?|music video)[^\]]*\]/gi, '').replace(/\s+/g, ' ').trim().slice(0, 300);
+  channel = String(channel).trim().slice(0, 120);
+  return [...new Set([title && channel ? `${title} ${channel}` : '', title, channel].filter(Boolean))];
+}
+export function selectRelated(cards, id, limit = 8) {
+  const seen = new Set([id]);
+  return (Array.isArray(cards) ? cards : []).filter(card => {
+    if (!card || typeof card.id !== 'string' || !card.title || seen.has(card.id)) return false;
+    seen.add(card.id); return true;
+  }).slice(0, limit);
+}
+export function createRelatedService({ next, search, local, collect = async () => {}, decorate = async cards => cards, now = Date.now, budgetMs = 18000 }) {
+  const cache = new Map(), pending = new Map();
+  async function load(seed, { refresh = false } = {}) {
+    const key = JSON.stringify([seed.id, seed.title, seed.channel]);
+    const cached = cache.get(key);
+    if (!refresh && cached && now() - cached.at < cached.ttl) return cached.result;
+    if (pending.has(key)) return pending.get(key);
+    const task = (async () => {
+      const deadline = now() + budgetMs, details = [];
+      let results = [], source = 'youtube';
+      try { results = selectRelated(await next(seed.id, Math.min(4000, budgetMs)), seed.id); }
+      catch (error) { details.push(String(error.message || error).slice(0, 300)); }
+      const queries = relatedQueries(seed);
+      for (const query of queries) {
+        if (results.length || now() >= deadline) break;
+        try { results = selectRelated(await search(query, Math.max(1, deadline - now())), seed.id); if (results.length) source = 'search'; }
+        catch (error) { details.push(String(error.message || error).slice(0, 300)); }
+      }
+      if (!results.length) {
+        source = 'catalog';
+        // Local results still work when YouTube blocks or times out. Broad queries
+        // avoid requiring every word of a long video title to be in another title.
+        const localQueries = [...new Set([seed.channel, seed.title?.split(/\s+/).slice(0, 3).join(' '), ...queries].filter(Boolean))];
+        for (const query of localQueries) {
+          try { results = selectRelated(await local(query), seed.id); } catch { /* try another local query */ }
+          if (results.length) break;
+        }
+      }
+      if (!results.length) return { ok: false, error: 'Related videos aren’t available right now. Please try again.', detail: details.join('\n') };
+      // Metadata/thumbnail persistence must not discard otherwise playable cards.
+      try { await collect(results); } catch { /* discovery remains usable */ }
+      try { results = await decorate(results); } catch { /* remote thumbnails remain */ }
+      const result = { ok: true, results, source, ...(source === 'catalog' ? { message: 'YouTube is unavailable. Showing related videos already known to this server.' } : {}) };
+      if (cache.size >= 500) cache.delete(cache.keys().next().value);
+      cache.set(key, { result, at: now(), ttl: source === 'catalog' ? 60000 : 600000 });
+      return result;
+    })().finally(() => pending.delete(key));
+    pending.set(key, task); return task;
+  }
+  return { load };
+}
+export function registerRelatedRoute(app, service) {
+  app.get('/api/related', async c => {
+    const id = (c.req.query('videoId') || '').trim();
+    if (!/^[\w-]{11}$/.test(id)) return c.json({ ok: false, error: 'Invalid video id.' }, 400);
+    const seed = { id, title: (c.req.query('title') || '').trim().slice(0, 300), channel: (c.req.query('channel') || '').trim().slice(0, 120) };
+    try {
+      const result = await service.load(seed, { refresh: c.req.query('refresh') === '1' });
+      return c.json(result, result.ok ? 200 : 503);
+    } catch { return c.json({ ok: false, error: 'Related videos aren’t available right now. Please try again.' }, 503); }
+  });
+}

server/related.test.js

@@ -0,0 +1,43 @@
+import { test, expect } from 'bun:test';
+import { Hono } from 'hono';
+import { createRelatedService, relatedQueries, selectRelated, registerRelatedRoute } from './related.js';
+const seed = { id: 'aaaaaaaaaaa', title: 'Grace (Official Lyric Video)', channel: 'Artist' };
+const card = { id: 'bbbbbbbbbbb', title: 'Another song', channel: 'Artist' };
+const fail = async () => { throw new Error('YouTube HTTP 429'); };
+test('fallback queries remove video boilerplate and deduplicate; cards exclude seed', () => {
+  expect(relatedQueries(seed)).toEqual(['Grace Artist', 'Grace', 'Artist']);
+  expect(selectRelated([{ ...card, id: seed.id }, card, card, null], seed.id)).toEqual([card]);
+});
+test('watch-next results bypass deep search and collect metadata', async () => {
+  let searches = 0, collected;
+  const service = createRelatedService({ next: async () => [card], search: async () => { searches++; return []; }, local: async () => [], collect: async cards => { collected = cards; } });
+  expect((await service.load(seed)).source).toBe('youtube'); expect(searches).toBe(0); expect(collected).toEqual([card]);
+});
+test('YouTube next failure and empty narrow query fall back to title search', async () => {
+  const queries = [];
+  const service = createRelatedService({ next: fail, search: async q => { queries.push(q); return q === 'Grace' ? [card] : []; }, local: async () => [] });
+  expect((await service.load(seed)).source).toBe('search'); expect(queries).toEqual(['Grace Artist', 'Grace']);
+});
+test('catalog fallback works without YouTube and metadata failures do not hide cards', async () => {
+  const service = createRelatedService({ next: fail, search: fail, local: async q => q === 'Artist' ? [card] : [], collect: fail });
+  const answer = await service.load(seed); expect(answer.ok).toBe(true); expect(answer.source).toBe('catalog'); expect(answer.results).toEqual([card]);
+});
+test('shared inflight request, successful cache, and explicit refresh', async () => {
+  let finish, calls=0;
+  const service=createRelatedService({next:async()=>{calls++;return new Promise(resolve=>{finish=resolve;});},search:fail,local:async()=>[]});
+  const a=service.load(seed),b=service.load(seed); finish([card]); await Promise.all([a,b]); expect(calls).toBe(1);
+  await service.load(seed); expect(calls).toBe(1);
+  const retry=service.load(seed,{refresh:true}); finish([card]); await retry; expect(calls).toBe(2);
+});
+test('failed responses are never cached, and exhausted budget goes straight to catalog', async () => {
+  let time=0, calls=0, searches=0;
+  const service=createRelatedService({now:()=>time,budgetMs:10,next:async()=>{calls++;time+=20;throw new Error('timeout');},search:async()=>{searches++;return[];},local:async()=>[]});
+  expect((await service.load(seed)).ok).toBe(false); expect((await service.load(seed)).ok).toBe(false);
+  expect(calls).toBe(2); expect(searches).toBe(0);
+});
+test('endpoint validates ids and provides a friendly retryable failure', async () => {
+  const app=new Hono();registerRelatedRoute(app,createRelatedService({next:fail,search:fail,local:async()=>[]}));
+  expect((await app.request('/api/related?videoId=bad')).status).toBe(400);
+  const response=await app.request(`/api/related?videoId=${seed.id}&title=Grace`); expect(response.status).toBe(503);
+  const body=await response.json(); expect(body.error).toContain('Please try again');expect(body.detail).toContain('429');
+});

server/server.js

@@ -61,6 +61,7 @@ import { registerIntakeRoutes } from './p2p-intake.js';
 import { createP2pHub, holdersPayload, createRehydrator } from './p2p-hub.js';
 import { sha256Range } from './hash.js';
 import * as innertube from './innertube.js';
+import { createRelatedService, registerRelatedRoute } from './related.js';
 import * as searchCacheDb from './search-cache.js';
 import { ingest as collectVideoMetadata, syncListening, linkListening, startThumbnails } from './video-catalog.js';
 import { registerCatalogRoutes } from './recommendations.js';
@@ -488,6 +489,25 @@ app.get('/api/search', async (c) => {
   }
 });
 
+// Eight related cards should not depend on a 200-result deep search succeeding.
+registerRelatedRoute(app, createRelatedService({
+  next: (id, timeoutMs) => process.env.SEARCH_INNERTUBE === '0' ? Promise.reject(new Error('InnerTube disabled')) : innertube.related(id, { timeoutMs }),
+  search: async (query, budgetMs) => {
+    const start = Date.now();
+    try {
+      if (process.env.SEARCH_INNERTUBE !== '0') {
+        const cards = await innertube.search(query, { timeoutMs: Math.min(3500, budgetMs) });
+        if (cards.length) return cards;
+      }
+    } catch { /* use the existing resilient yt-dlp search path */ }
+    const remaining = budgetMs - (Date.now() - start);
+    if (remaining <= 0) throw new Error('Related search timed out');
+    return parseCards(await runYtdlpResilient([`ytsearch12:${query}`, '--dump-json', '--flat-playlist', '--no-warnings', '--ignore-errors', '--socket-timeout', '4', '--retries', '1'], { signal: AbortSignal.timeout(Math.min(6000, remaining)) }));
+  },
+  local: query => searchCacheDb.searchVideos(query, 20),
+  collect: cards => collectVideoMetadata(cards, 'related'),
+}));
+
 // GET /api/channel?c=<channel>
 // Channel listing that works for every kind of channel, not one-off cases:
 //   1. /videos, /streams, /shorts tabs        — normal channels