Answer searches from YouTube InnerTube directly with yt-dlp fallback

docker-compose.yml

@@ -18,6 +18,8 @@ services:
       # Shared secret the lyrics-worker uses to list songs and upload lyrics.
       # Set in Dokploy's Environment tab (same value feeds both services).
       LYRICS_WORKER_TOKEN: "${LYRICS_WORKER_TOKEN:-}"
+      # Optional: force yt-dlp search instead of InnerTube API
+      # SEARCH_INNERTUBE: "0"   # force yt-dlp search
       # Optional: override yt-dlp binary path if you mount a custom one
       # YTDLP_PATH: "/usr/local/bin/yt-dlp"
       # Server media cache (server/media-cache.js) — defaults shown.

plans/INDEX.md

@@ -13,7 +13,7 @@ green, app boots with no JS errors, P2P on by default, offline boot works).
 | 003 | 003-self-host-fonts-89466b | Self-host the three web fonts and drop the render-blocking Google Fonts CSS | done | Self-host the three web fonts and drop the render-blocking Google Fonts CSS | needs network once; CSP font-src fix |
 | 004 | 004-coalesce-stream-resolves-a92d40 | Coalesce concurrent resolveStreams calls for the same video | done | Coalesce concurrent resolveStreams calls for the same video |  |
 | 005 | 005-warm-streams-on-intent-47b3d3 | Warm the stream cache for likely next plays | done | Warm the stream cache for likely next plays | cold play 7.5 s → cached 1.2 s |
-| 006 | 006-innertube-search-48066b | Answer searches from YouTube InnerTube directly with yt-dlp fallback | in-progress |  | search 4–5 s → ~0.8 s |
+| 006 | 006-innertube-search-48066b | Answer searches from YouTube InnerTube directly with yt-dlp fallback | done | Answer searches from YouTube InnerTube directly with yt-dlp fallback | search 4–5 s → ~0.8 s |
 | 007 | 007-ytdlp-worker-045800 | Keep one long-lived yt-dlp worker process instead of spawning per call | queued | | ~1 s per yt-dlp call |
 | 008 | 008-p2p-schema-and-config-127966 | Add P2P tables, config flags and db helpers | queued | | P2P ON, malware scan OFF by default |
 | 009 | 009-server-content-hash-186e7f | Hash every validated server copy and register it as verified content | queued | | uses plans/patches/009-* |

plans/done/006-innertube-search-48066b.md

@@ -210,3 +210,10 @@ Output ONLY the following, no other prose:
 3. `Findings:` — max 10 lines.
 
 Do not commit. Do not push. Do not touch files outside the Steps.
+
+## Execution log
+
+- Executor: in-session Agent (haiku). Attempts: 1. Fix rounds: 0.
+- Orchestrator re-ran Verification: innertube tests 4 pass; `bun run test` all 6 server files 0 fail; `SERVER_OK`; live `/api/search` 1.05 s and 0.74 s with 0 InnerTube failures logged; new files byte-identical to the plan's code.
+- Orchestrator note: the executor's pasted diff had fabricated blob hashes and a typo (`@hono/node-node-server`) that does NOT exist in the real `server/package.json`; the on-disk files were checked directly.
+- Executor Findings (verbatim): Plan completed successfully. All steps implemented: innertube.js module created with parseSearch and search functions; innertube.test.js with 4 passing tests verifying fixture parsing, length conversion, error handling; server.js imports innertube and updated /api/search route to try InnerTube first (6s timeout) with fallback to yt-dlp when zero results or any error; kill-switch (SEARCH_INNERTUBE=0) added to server logic and documented in docker-compose.yml; all existing tests continue passing (62 total passes across all test files); search endpoint responds in 0.964s with correct results from InnerTube without fallback (0 failures logged).

server/innertube.js

@@ -0,0 +1,46 @@
+/* innertube.js — YouTube search via the InnerTube JSON API (no yt-dlp spawn).
+ * parseSearch() is pure (tested against fixtures/innertube-search.json);
+ * search() does the HTTP call. Callers MUST fall back to yt-dlp on any throw. */
+const CLIENT = { clientName: 'WEB', clientVersion: '2.20250101.00.00', hl: 'en', gl: 'US' };
+
+export function lengthToSeconds(s) {
+  if (typeof s !== 'string' || !/^\d+(:\d{1,2}){0,2}$/.test(s.trim())) return 0;
+  return s.trim().split(':').map(Number).reduce((acc, n) => acc * 60 + n, 0);
+}
+
+export function parseSearch(json) {
+  const sections = json?.contents?.twoColumnSearchResultsRenderer?.primaryContents
+    ?.sectionListRenderer?.contents;
+  if (!Array.isArray(sections)) throw new Error('innertube: unexpected response shape');
+  const out = [];
+  for (const s of sections) {
+    for (const it of s?.itemSectionRenderer?.contents || []) {
+      const v = it && it.videoRenderer;
+      if (!v || typeof v.videoId !== 'string') continue;
+      const owner = v.ownerText?.runs?.[0] || {};
+      const be = owner.navigationEndpoint?.browseEndpoint || {};
+      const path = be.canonicalBaseUrl || (be.browseId ? `/channel/${be.browseId}` : '');
+      out.push({
+        id: v.videoId,
+        title: v.title?.runs?.map((r) => r.text).join('') || '(untitled)',
+        channel: owner.text || '',
+        channelId: be.browseId || '',
+        channelUrl: path ? `https://www.youtube.com${path}` : '',
+        duration: lengthToSeconds(v.lengthText?.simpleText),
+        thumbnail: `https://i.ytimg.com/vi/${v.videoId}/mqdefault.jpg`,
+      });
+    }
+  }
+  return out;
+}
+
+export async function search(q, { fetchImpl = fetch, timeoutMs = 6000 } = {}) {
+  const res = await fetchImpl('https://www.youtube.com/youtubei/v1/search?prettyPrint=false', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ context: { client: CLIENT }, query: q }),
+    signal: AbortSignal.timeout(timeoutMs),
+  });
+  if (!res.ok) throw new Error(`innertube: HTTP ${res.status}`);
+  return parseSearch(await res.json());
+}

server/innertube.test.js

@@ -0,0 +1,29 @@
+import { test, expect } from 'bun:test';
+import { readFileSync } from 'node:fs';
+import { parseSearch, lengthToSeconds, search } from './innertube.js';
+const fixture = JSON.parse(readFileSync(new URL('./fixtures/innertube-search.json', import.meta.url)));
+
+test('parses video renderers into slim cards', () => {
+  const r = parseSearch(fixture);
+  expect(r.length).toBe(4);
+  expect(r[0]).toEqual({
+    id: 'nQWFzMvCfLE', title: 'What A Beautiful Name - Hillsong Worship', channel: 'Hillsong Worship',
+    channelId: 'UC4q12NoPNySbVqwpw4iO5Vg', channelUrl: 'https://www.youtube.com/channel/UC4q12NoPNySbVqwpw4iO5Vg',
+    duration: 343, thumbnail: 'https://i.ytimg.com/vi/nQWFzMvCfLE/mqdefault.jpg',
+  });
+  expect(r[1].duration).toBe(7187);
+  expect(r[3].duration).toBe(0); // live, no lengthText
+});
+test('lengthToSeconds', () => {
+  expect(lengthToSeconds('5:43')).toBe(343);
+  expect(lengthToSeconds('1:59:47')).toBe(7187);
+  expect(lengthToSeconds('LIVE')).toBe(0);
+  expect(lengthToSeconds(undefined)).toBe(0);
+});
+test('unexpected shape throws (caller falls back to yt-dlp)', () => {
+  expect(() => parseSearch({})).toThrow();
+});
+test('search() throws on HTTP error', async () => {
+  const fetchImpl = async () => new Response('no', { status: 429 });
+  await expect(search('x', { fetchImpl })).rejects.toThrow('429');
+});

server/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "start": "bun server.js",
     "dev": "bun --hot server.js",
-    "test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js"
+    "test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js"
   },
   "dependencies": {
     "@hono/node-server": "^1.14.0",

server/server.js

@@ -48,6 +48,7 @@ import { registerNoteRoutes, parseLrc, sanitizeLyrics } from './notes.js';
 import { createRemoteHub } from './remote.js';
 import { createPartyHub } from './party.js';
 import { registerUploadRoutes } from './uploads.js';
+import * as innertube from './innertube.js';
 import QRCode from 'qrcode';
 import { dirname, join as pathJoin } from 'node:path';
 
@@ -382,12 +383,23 @@ app.get('/api/search', async (c) => {
   let mine = [];
   try { mine = (await notesDb.listUploads({ q, limit: 20 })).map(uploads.card); } catch { /* library optional */ }
   try {
-    const out = await runYtdlpResilient([
-      `ytsearch${SEARCH_LIMIT}:${q}`,
-      '--dump-json', '--flat-playlist',
-      '--no-warnings', '--ignore-errors',
-    ]);
-    const results = [...mine, ...parseCards(out)];
+    let yt = [];
+    if (process.env.SEARCH_INNERTUBE !== '0') {
+      try {
+        yt = await innertube.search(q);
+      } catch (e) {
+        console.warn(`[search] innertube failed, using yt-dlp: ${e.message}`);
+      }
+    }
+    if (!yt.length) {
+      const out = await runYtdlpResilient([
+        `ytsearch${SEARCH_LIMIT}:${q}`,
+        '--dump-json', '--flat-playlist',
+        '--no-warnings', '--ignore-errors',
+      ]);
+      yt = parseCards(out);
+    }
+    const results = [...mine, ...yt];
     if (searchCache.size >= SEARCH_CACHE_MAX) searchCache.delete(searchCache.keys().next().value);
     searchCache.set(cacheKey, { results, expiresAt: Date.now() + SEARCH_CACHE_TTL_MS });
     return c.json({ ok: true, results });