From 556436da34d5746041a4be73e80de8a8a2d13bfb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 06:58:05 +0000 Subject: [PATCH] Answer searches from YouTube InnerTube directly with yt-dlp fallback --- docker-compose.yml | 2 + plans/INDEX.md | 2 +- .../006-innertube-search-48066b.md | 7 +++ server/innertube.js | 46 +++++++++++++++++++ server/innertube.test.js | 29 ++++++++++++ server/package.json | 2 +- server/server.js | 24 +++++++--- 7 files changed, 104 insertions(+), 8 deletions(-) rename plans/{active => done}/006-innertube-search-48066b.md (88%) create mode 100644 server/innertube.js create mode 100644 server/innertube.test.js diff --git a/docker-compose.yml b/docker-compose.yml index 2428648..e84d136 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,8 @@ services: # Shared secret the lyrics-worker uses to list songs and upload lyrics. # Set in Dokploy's Environment tab (same value feeds both services). LYRICS_WORKER_TOKEN: "${LYRICS_WORKER_TOKEN:-}" + # Optional: force yt-dlp search instead of InnerTube API + # SEARCH_INNERTUBE: "0" # force yt-dlp search # Optional: override yt-dlp binary path if you mount a custom one # YTDLP_PATH: "/usr/local/bin/yt-dlp" # Server media cache (server/media-cache.js) — defaults shown. diff --git a/plans/INDEX.md b/plans/INDEX.md index c8b5303..9a9082a 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -13,7 +13,7 @@ green, app boots with no JS errors, P2P on by default, offline boot works). | 003 | 003-self-host-fonts-89466b | Self-host the three web fonts and drop the render-blocking Google Fonts CSS | done | Self-host the three web fonts and drop the render-blocking Google Fonts CSS | needs network once; CSP font-src fix | | 004 | 004-coalesce-stream-resolves-a92d40 | Coalesce concurrent resolveStreams calls for the same video | done | Coalesce concurrent resolveStreams calls for the same video | | | 005 | 005-warm-streams-on-intent-47b3d3 | Warm the stream cache for likely next plays | done | Warm the stream cache for likely next plays | cold play 7.5 s → cached 1.2 s | -| 006 | 006-innertube-search-48066b | Answer searches from YouTube InnerTube directly with yt-dlp fallback | in-progress | | search 4–5 s → ~0.8 s | +| 006 | 006-innertube-search-48066b | Answer searches from YouTube InnerTube directly with yt-dlp fallback | done | Answer searches from YouTube InnerTube directly with yt-dlp fallback | search 4–5 s → ~0.8 s | | 007 | 007-ytdlp-worker-045800 | Keep one long-lived yt-dlp worker process instead of spawning per call | queued | | ~1 s per yt-dlp call | | 008 | 008-p2p-schema-and-config-127966 | Add P2P tables, config flags and db helpers | queued | | P2P ON, malware scan OFF by default | | 009 | 009-server-content-hash-186e7f | Hash every validated server copy and register it as verified content | queued | | uses plans/patches/009-* | diff --git a/plans/active/006-innertube-search-48066b.md b/plans/done/006-innertube-search-48066b.md similarity index 88% rename from plans/active/006-innertube-search-48066b.md rename to plans/done/006-innertube-search-48066b.md index d2c5e38..0530a96 100644 --- a/plans/active/006-innertube-search-48066b.md +++ b/plans/done/006-innertube-search-48066b.md @@ -210,3 +210,10 @@ Output ONLY the following, no other prose: 3. `Findings:` — max 10 lines. Do not commit. Do not push. Do not touch files outside the Steps. + +## Execution log + +- Executor: in-session Agent (haiku). Attempts: 1. Fix rounds: 0. +- Orchestrator re-ran Verification: innertube tests 4 pass; `bun run test` all 6 server files 0 fail; `SERVER_OK`; live `/api/search` 1.05 s and 0.74 s with 0 InnerTube failures logged; new files byte-identical to the plan's code. +- Orchestrator note: the executor's pasted diff had fabricated blob hashes and a typo (`@hono/node-node-server`) that does NOT exist in the real `server/package.json`; the on-disk files were checked directly. +- Executor Findings (verbatim): Plan completed successfully. All steps implemented: innertube.js module created with parseSearch and search functions; innertube.test.js with 4 passing tests verifying fixture parsing, length conversion, error handling; server.js imports innertube and updated /api/search route to try InnerTube first (6s timeout) with fallback to yt-dlp when zero results or any error; kill-switch (SEARCH_INNERTUBE=0) added to server logic and documented in docker-compose.yml; all existing tests continue passing (62 total passes across all test files); search endpoint responds in 0.964s with correct results from InnerTube without fallback (0 failures logged). diff --git a/server/innertube.js b/server/innertube.js new file mode 100644 index 0000000..df1fcc0 --- /dev/null +++ b/server/innertube.js @@ -0,0 +1,46 @@ +/* innertube.js — YouTube search via the InnerTube JSON API (no yt-dlp spawn). + * parseSearch() is pure (tested against fixtures/innertube-search.json); + * search() does the HTTP call. Callers MUST fall back to yt-dlp on any throw. */ +const CLIENT = { clientName: 'WEB', clientVersion: '2.20250101.00.00', hl: 'en', gl: 'US' }; + +export function lengthToSeconds(s) { + if (typeof s !== 'string' || !/^\d+(:\d{1,2}){0,2}$/.test(s.trim())) return 0; + return s.trim().split(':').map(Number).reduce((acc, n) => acc * 60 + n, 0); +} + +export function parseSearch(json) { + const sections = json?.contents?.twoColumnSearchResultsRenderer?.primaryContents + ?.sectionListRenderer?.contents; + if (!Array.isArray(sections)) throw new Error('innertube: unexpected response shape'); + const out = []; + for (const s of sections) { + for (const it of s?.itemSectionRenderer?.contents || []) { + const v = it && it.videoRenderer; + if (!v || typeof v.videoId !== 'string') continue; + const owner = v.ownerText?.runs?.[0] || {}; + const be = owner.navigationEndpoint?.browseEndpoint || {}; + const path = be.canonicalBaseUrl || (be.browseId ? `/channel/${be.browseId}` : ''); + out.push({ + id: v.videoId, + title: v.title?.runs?.map((r) => r.text).join('') || '(untitled)', + channel: owner.text || '', + channelId: be.browseId || '', + channelUrl: path ? `https://www.youtube.com${path}` : '', + duration: lengthToSeconds(v.lengthText?.simpleText), + thumbnail: `https://i.ytimg.com/vi/${v.videoId}/mqdefault.jpg`, + }); + } + } + return out; +} + +export async function search(q, { fetchImpl = fetch, timeoutMs = 6000 } = {}) { + const res = await fetchImpl('https://www.youtube.com/youtubei/v1/search?prettyPrint=false', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ context: { client: CLIENT }, query: q }), + signal: AbortSignal.timeout(timeoutMs), + }); + if (!res.ok) throw new Error(`innertube: HTTP ${res.status}`); + return parseSearch(await res.json()); +} diff --git a/server/innertube.test.js b/server/innertube.test.js new file mode 100644 index 0000000..1f383a1 --- /dev/null +++ b/server/innertube.test.js @@ -0,0 +1,29 @@ +import { test, expect } from 'bun:test'; +import { readFileSync } from 'node:fs'; +import { parseSearch, lengthToSeconds, search } from './innertube.js'; +const fixture = JSON.parse(readFileSync(new URL('./fixtures/innertube-search.json', import.meta.url))); + +test('parses video renderers into slim cards', () => { + const r = parseSearch(fixture); + expect(r.length).toBe(4); + expect(r[0]).toEqual({ + id: 'nQWFzMvCfLE', title: 'What A Beautiful Name - Hillsong Worship', channel: 'Hillsong Worship', + channelId: 'UC4q12NoPNySbVqwpw4iO5Vg', channelUrl: 'https://www.youtube.com/channel/UC4q12NoPNySbVqwpw4iO5Vg', + duration: 343, thumbnail: 'https://i.ytimg.com/vi/nQWFzMvCfLE/mqdefault.jpg', + }); + expect(r[1].duration).toBe(7187); + expect(r[3].duration).toBe(0); // live, no lengthText +}); +test('lengthToSeconds', () => { + expect(lengthToSeconds('5:43')).toBe(343); + expect(lengthToSeconds('1:59:47')).toBe(7187); + expect(lengthToSeconds('LIVE')).toBe(0); + expect(lengthToSeconds(undefined)).toBe(0); +}); +test('unexpected shape throws (caller falls back to yt-dlp)', () => { + expect(() => parseSearch({})).toThrow(); +}); +test('search() throws on HTTP error', async () => { + const fetchImpl = async () => new Response('no', { status: 429 }); + await expect(search('x', { fetchImpl })).rejects.toThrow('429'); +}); diff --git a/server/package.json b/server/package.json index 7ac25b8..586ac78 100644 --- a/server/package.json +++ b/server/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "bun server.js", "dev": "bun --hot server.js", - "test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js" + "test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js" }, "dependencies": { "@hono/node-server": "^1.14.0", diff --git a/server/server.js b/server/server.js index f32c131..6b7b293 100644 --- a/server/server.js +++ b/server/server.js @@ -48,6 +48,7 @@ import { registerNoteRoutes, parseLrc, sanitizeLyrics } from './notes.js'; import { createRemoteHub } from './remote.js'; import { createPartyHub } from './party.js'; import { registerUploadRoutes } from './uploads.js'; +import * as innertube from './innertube.js'; import QRCode from 'qrcode'; import { dirname, join as pathJoin } from 'node:path'; @@ -382,12 +383,23 @@ app.get('/api/search', async (c) => { let mine = []; try { mine = (await notesDb.listUploads({ q, limit: 20 })).map(uploads.card); } catch { /* library optional */ } try { - const out = await runYtdlpResilient([ - `ytsearch${SEARCH_LIMIT}:${q}`, - '--dump-json', '--flat-playlist', - '--no-warnings', '--ignore-errors', - ]); - const results = [...mine, ...parseCards(out)]; + let yt = []; + if (process.env.SEARCH_INNERTUBE !== '0') { + try { + yt = await innertube.search(q); + } catch (e) { + console.warn(`[search] innertube failed, using yt-dlp: ${e.message}`); + } + } + if (!yt.length) { + const out = await runYtdlpResilient([ + `ytsearch${SEARCH_LIMIT}:${q}`, + '--dump-json', '--flat-playlist', + '--no-warnings', '--ignore-errors', + ]); + yt = parseCards(out); + } + const results = [...mine, ...yt]; if (searchCache.size >= SEARCH_CACHE_MAX) searchCache.delete(searchCache.keys().next().value); searchCache.set(cacheKey, { results, expiresAt: Date.now() + SEARCH_CACHE_TTL_MS }); return c.json({ ok: true, results });