From c629ff583616effab39a8a4152a6c79e229e9d1b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 07:38:47 +0000 Subject: [PATCH] Register devices and report verified holdings to the server --- frontend/app.js | 6 + frontend/hash-worker.js | 26 +++ frontend/index.html | 1 + frontend/opfs.js | 13 ++ frontend/p2p-client.js | 189 ++++++++++++++++++ frontend/sw.js | 2 + plans/INDEX.md | 2 +- ...013-device-identity-and-holdings-3ba493.md | 6 + server/p2p-db.js | 10 + server/p2p-routes.js | 133 ++++++++++++ server/p2p-routes.test.js | 111 ++++++++++ server/package.json | 2 +- server/server.js | 14 ++ 13 files changed, 513 insertions(+), 2 deletions(-) create mode 100644 frontend/hash-worker.js create mode 100644 frontend/p2p-client.js rename plans/{active => done}/013-device-identity-and-holdings-3ba493.md (96%) create mode 100644 server/p2p-routes.js create mode 100644 server/p2p-routes.test.js diff --git a/frontend/app.js b/frontend/app.js index 3956cb2..595d5e2 100755 --- a/frontend/app.js +++ b/frontend/app.js @@ -353,6 +353,8 @@ function toAssetUrl(path) { // ---------- State ---------- const DEFAULT_SETTINGS = { quality: 'auto', volume: 1, audioOnly: false, autoPreload: true, + p2pShare: true, // P2P (docs/p2p-architecture.md): share my saved videos with other devices — ON by default + p2pReceive: true, // fetch from other devices when YouTube and the server can't — ON by default saveBeforePlay: false, // download (server-muxed single file) before playing instead of streaming repeatMode: 'off', // 'off' | 'all' — repeat the playing list when it ends loopOne: false, // repeat the single current video @@ -9874,6 +9876,10 @@ async function boot() { // Learn what's already cached, then top up any playlist videos that aren't. await refreshCachedIds(); startCacheResyncWatch(); + // Peer-to-peer: report what this device holds (on by default; settings.p2pShare). + if (WEB && window.P2PClient) { + window.P2PClient.start({ getSettings: () => data.settings, getProfile: () => (data.profile && data.profile.name) || '' }); + } data.playlists.forEach(preloadPlaylist); preloadPinnedPlaylists(); // Not awaited — artwork backfill must never delay first paint. diff --git a/frontend/hash-worker.js b/frontend/hash-worker.js new file mode 100644 index 0000000..56a3e62 --- /dev/null +++ b/frontend/hash-worker.js @@ -0,0 +1,26 @@ +/* ============================================================================ + * hash-worker.js — SHA-256 of a file already saved in OPFS, off the main + * thread, read in 4 MiB slices (never the whole video in memory). + * + * In: { name } file name under OPFS videos/ (e.g. "abc.mp4") + * Out: { ok: true, sha256, size } | { ok: false, error } + * ========================================================================== */ +'use strict'; +importScripts('/sha256.js'); + +self.onmessage = async (e) => { + const { name } = e.data || {}; + try { + const root = await navigator.storage.getDirectory(); + const dir = await root.getDirectoryHandle('videos'); + const file = await (await dir.getFileHandle(String(name))).getFile(); + const h = self.Sha256.create(); + const STEP = 4 * 1024 * 1024; + for (let pos = 0; pos < file.size; pos += STEP) { + h.update(new Uint8Array(await file.slice(pos, pos + STEP).arrayBuffer())); + } + self.postMessage({ ok: true, sha256: h.hex(), size: file.size }); + } catch (err) { + self.postMessage({ ok: false, error: err && err.message ? err.message : String(err) }); + } +}; diff --git a/frontend/index.html b/frontend/index.html index f0ddef3..0b5aa2e 100755 --- a/frontend/index.html +++ b/frontend/index.html @@ -563,6 +563,7 @@ + diff --git a/frontend/opfs.js b/frontend/opfs.js index e1a18a4..3b12736 100644 --- a/frontend/opfs.js +++ b/frontend/opfs.js @@ -108,6 +108,19 @@ } }, + // Bytes [offset, offset+length) of a saved video — answers the server's + // P2P range challenges (p2p-client.js). null when the file is missing. + async readRange(videoId, offset, length) { + try { + const found = await findHandle(videoId); + if (!found) return null; + const file = await found[0].getFile(); + return new Uint8Array(await file.slice(offset, offset + length).arrayBuffer()); + } catch { + return null; + } + }, + revokeUrl(url) { if (url && _blobUrls.has(url)) { URL.revokeObjectURL(url); diff --git a/frontend/p2p-client.js b/frontend/p2p-client.js new file mode 100644 index 0000000..b999763 --- /dev/null +++ b/frontend/p2p-client.js @@ -0,0 +1,189 @@ +/* ============================================================================ + * p2p-client.js — this device's side of peer-to-peer sharing + * (docs/p2p-architecture.md flows 2–3). window.P2PClient. + * + * start({ getSettings, getProfile }) called once from app.js boot() + * changed() a save/delete happened → re-report soon + * device() { deviceId, secret } or null + * authHeaders() { 'X-Device': … } for other P2P calls + * + * What it does, in order, each sync: + * 1. registers the device once (localStorage ytpDevice) + * 2. reconciles DeviceDB with the files really in OPFS (a record without a + * file is dropped; a file without a record is added as 'unhashed') + * 3. hashes 'unhashed' files and files not re-checked for 30 days, one at a + * time in hash-worker.js + * 4. reports the FULL holdings list (empty when sharing is off), answers the + * server's range challenges, and marks accepted files 'verified' + * P2P is ON by default: sharing runs unless settings.p2pShare === false or the + * server says P2P is disabled. + * ========================================================================== */ +(function () { + 'use strict'; + + const KEY = 'ytpDevice'; + const REHASH_MS = 30 * 24 * 3600_000; + const RESYNC_MS = 6 * 3600_000; + let hooks = { getSettings: () => ({}), getProfile: () => '' }; + let serverCfg = null; + let running = null; + let again = false; + let timer = null; + let lastSync = 0; + + function device() { + try { + const d = JSON.parse(localStorage.getItem(KEY) || 'null'); + return d && /^dev_[0-9a-f]{16}$/.test(d.deviceId) && /^[0-9a-f]{64}$/.test(d.secret) ? d : null; + } catch { return null; } + } + const authHeaders = () => { const d = device(); return d ? { 'X-Device': d.deviceId + '.' + d.secret } : {}; }; + + async function config() { + if (serverCfg) return serverCfg; + try { + const j = await (await fetch('/api/p2p/config')).json(); + if (j && j.ok) serverCfg = j; + } catch { /* offline — try again next sync */ } + return serverCfg; + } + + async function ensureDevice() { + const have = device(); + if (have) return have; + const r = await fetch('/api/p2p/device', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ fingerprint: window.getFingerprint ? window.getFingerprint() : '', profile: hooks.getProfile() || '' }), + }); + const j = await r.json(); + if (!j || !j.ok) throw new Error((j && j.error) || 'device registration failed'); + const d = { deviceId: j.deviceId, secret: j.secret }; + try { localStorage.setItem(KEY, JSON.stringify(d)); } catch { /* storage blocked */ } + return d; + } + + function hashInWorker(name) { + return new Promise((resolve) => { + let w; + try { w = new Worker('/hash-worker.js'); } catch { resolve(null); return; } + w.onmessage = (e) => { w.terminate(); resolve(e.data && e.data.ok ? e.data : null); }; + w.onerror = () => { w.terminate(); resolve(null); }; + w.postMessage({ name }); + }); + } + + async function sha256Range(videoId, offset, length) { + const bytes = window.OPFS && window.OPFS.readRange ? await window.OPFS.readRange(videoId, offset, length) : null; + if (!bytes) return null; + return window.Sha256.hex(bytes); + } + + // DeviceDB ⇄ OPFS. Returns the records that describe a real file. + async function reconcile() { + const files = await window.OPFS.listVideos({ strict: true }); + const byId = new Map(files.map((f) => [f.id, f])); + const recs = await window.DeviceDB.listFiles(); + const out = []; + for (const r of recs) { + const f = byId.get(r.videoId); + if (!f) { await window.DeviceDB.deleteFile(r.videoId); continue; } + if (f.size !== r.size && r.size) { r.cid = null; r.state = 'unhashed'; r.size = f.size; await window.DeviceDB.putFile(r); } + r.name = f.name; + out.push(r); + byId.delete(r.videoId); + } + for (const f of byId.values()) { + if (String(f.id).startsWith('edit_')) continue; // edited cuts are never shared + const r = { videoId: f.id, cid: null, size: f.size, savedAt: Date.now(), lastCheckedAt: 0, state: 'unhashed' }; + await window.DeviceDB.putFile(r); + out.push({ ...r, name: f.name }); + } + return out; + } + + async function hashPending(recs) { + const now = Date.now(); + for (const r of recs) { + if (r.state !== 'unhashed' && now - (r.lastCheckedAt || 0) < REHASH_MS) continue; + const h = await hashInWorker(r.name); + if (!h) continue; + const changedCid = h.sha256 !== r.cid; + r.cid = h.sha256; + r.size = h.size; + r.lastCheckedAt = Date.now(); + if (changedCid || r.state === 'unhashed') r.state = 'unverified'; + await window.DeviceDB.putFile(r); + } + } + + async function report(recs, dev) { + const share = hooks.getSettings().p2pShare !== false; + const items = recs.filter((r) => r.cid).map((r) => ({ cid: r.cid, size: r.size, videoId: r.videoId })); + const r = await fetch('/api/p2p/holdings', { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Device': dev.deviceId + '.' + dev.secret }, + body: JSON.stringify({ share, items: share ? items : [], profile: hooks.getProfile() || '' }), + }); + if (r.status === 401) { try { localStorage.removeItem(KEY); } catch { /* ignore */ } return null; } + const j = await r.json(); + if (!j || !j.ok) return null; + const accepted = new Set(j.accepted || []); + for (const rec of recs) { + if (rec.cid && accepted.has(rec.cid) && rec.state !== 'verified') { rec.state = 'verified'; await window.DeviceDB.putFile(rec); } + } + const answers = []; + for (const ch of j.challenges || []) { + const rec = recs.find((x) => x.cid === ch.cid); + if (!rec) continue; + const hex = await sha256Range(rec.videoId, ch.offset, ch.length); + if (hex) answers.push({ ...ch, sha256: hex }); + } + if (answers.length) { + await fetch('/api/p2p/challenge', { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Device': dev.deviceId + '.' + dev.secret }, + body: JSON.stringify({ answers }), + }).catch(() => {}); + } + return j; + } + + async function syncOnce() { + if (!window.OPFS || !window.OPFS.isSupported() || !window.DeviceDB || !window.Sha256) return null; + if (navigator.onLine === false) return null; + const cfg = await config(); + if (!cfg || !cfg.enabled) return null; + const dev = await ensureDevice(); + const recs = await reconcile(); + await hashPending(recs); + const res = await report(recs, dev); + lastSync = Date.now(); + return res; + } + + // One sync at a time; a request during a sync schedules exactly one more. + function sync() { + if (running) { again = true; return running; } + running = syncOnce().catch(() => null).finally(() => { + running = null; + if (again) { again = false; sync(); } + }); + return running; + } + + function changed() { + clearTimeout(timer); + timer = setTimeout(sync, 5000); + } + + function start(h) { + hooks = { ...hooks, ...(h || {}) }; + const idle = window.requestIdleCallback || ((fn) => setTimeout(fn, 1)); + setTimeout(() => idle(() => sync()), 8000); + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'visible' && Date.now() - lastSync > RESYNC_MS) sync(); + }); + } + + window.P2PClient = { start, changed, sync, device, authHeaders, config }; +}()); diff --git a/frontend/sw.js b/frontend/sw.js index f51c99b..5f7245e 100644 --- a/frontend/sw.js +++ b/frontend/sw.js @@ -66,6 +66,8 @@ const SHELL = [ '/stats-core.js', '/sha256.js', '/device-db.js', + '/hash-worker.js', + '/p2p-client.js', '/app.js', '/manifest.webmanifest', '/icons/icon-192.png', diff --git a/plans/INDEX.md b/plans/INDEX.md index 7c6e251..a014eb7 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -20,7 +20,7 @@ green, app boots with no JS errors, P2P on by default, offline boot works). | 010 | 010-views-and-retention-d0c6ca | Count views and evict server copies by retention criteria before LRU | done | Count views and evict server copies by retention criteria before LRU | | | 011 | 011-browser-sha256-e1793d | Add an incremental SHA-256 library for the browser and node tests | done | Add an incremental SHA-256 library for the browser and node tests | | | 012 | 012-device-file-registry-288d55 | Add the on-device IndexedDB file registry and hash saves while downloading | done | Add the on-device IndexedDB file registry and hash saves while downloading | browser harness | -| 013 | 013-device-identity-and-holdings-3ba493 | Register devices and report verified holdings to the server | in-progress | | persistent holders, no TTL | +| 013 | 013-device-identity-and-holdings-3ba493 | Register devices and report verified holdings to the server | done | Register devices and report verified holdings to the server | persistent holders, no TTL | | 014 | 014-p2p-presence-hub-ceced8 | Add the /ws/p2p presence and signalling hub and the holders endpoint | queued | | stale flag, never hidden | | 015 | 015-availability-ui-and-settings-3b9397 | Show peer availability with stale markers and add Sharing settings | queued | | | | 016 | 016-intake-and-server-verification-cfe031 | Let a device hand a file to the server for hashing and validation | queued | | needs ffmpeg for tests | diff --git a/plans/active/013-device-identity-and-holdings-3ba493.md b/plans/done/013-device-identity-and-holdings-3ba493.md similarity index 96% rename from plans/active/013-device-identity-and-holdings-3ba493.md rename to plans/done/013-device-identity-and-holdings-3ba493.md index 8a89106..a738199 100644 --- a/plans/active/013-device-identity-and-holdings-3ba493.md +++ b/plans/done/013-device-identity-and-holdings-3ba493.md @@ -620,3 +620,9 @@ self.onmessage = async (e) => { window.P2PClient = { start, changed, sync, device, authHeaders, config }; }()); ``` + +## Execution log + +- Executor: in-session Agent (haiku). Attempts: 1. Fix rounds: 0. +- Orchestrator re-ran Verification: `p2p-routes.js`, `p2p-routes.test.js`, `hash-worker.js`, `p2p-client.js`, `opfs.js` byte-identical to the pre-tested versions; routes tests 6 pass; all 12 server test files 0 fail; `SERVER_OK`; frontend syntax ok; 56 frontend tests pass; browser check in Chromium reproduced the expected JSON (accepted 1, unknown 1, challenge passed -> trust `challenged`, share off -> 0 holders, device registered). No leftover processes. +- Executor Findings (verbatim): All 12 steps completed successfully. P2P routes pass 6/6 tests. Server builds without errors. Frontend syntax check passes. All 56 frontend unit tests pass. Browser check returns expected JSON with device registration, holdings acceptance, unknown detection, challenge completion, and share-off withdrawal. git apply patch for OPFS.readRange applied cleanly. New files created: p2p-routes.js, p2p-routes.test.js, hash-worker.js, p2p-client.js match appendix specifications exactly. diff --git a/server/p2p-db.js b/server/p2p-db.js index d10551b..7a7784f 100644 --- a/server/p2p-db.js +++ b/server/p2p-db.js @@ -63,6 +63,7 @@ export async function initP2pSchema() { // media_cache.sha256 — the cid of the current ..mp4 (plan 009). try { await db.execute('ALTER TABLE media_cache ADD COLUMN sha256 TEXT'); } catch (e) { if (!/duplicate column/i.test(String(e.message))) throw e; } + await db.execute('CREATE INDEX IF NOT EXISTS idx_media_sha256 ON media_cache (sha256)'); } const rowsOf = (r) => r.rows.map((row) => { @@ -258,3 +259,12 @@ export async function listMediaEvictionOrder({ now, keepMinViews, keepDays, keep || (a.last_access - b.last_access)); return rows; } + +// The server's own ready copy whose mp4 has this content id (or null). +export async function findMediaByCid(cid) { + const r = await db.execute({ + sql: "SELECT video_id, gen FROM media_cache WHERE sha256 = ? AND status = 'ready' LIMIT 1", + args: [cid], + }); + return rowsOf(r)[0] || null; +} diff --git a/server/p2p-routes.js b/server/p2p-routes.js new file mode 100644 index 0000000..aa83d7a --- /dev/null +++ b/server/p2p-routes.js @@ -0,0 +1,133 @@ +/* ============================================================================ + * p2p-routes.js — device registration + holdings (docs/p2p-architecture.md + * flows 3 and 5). Mounted by server.js; every route answers 404 when + * P2P_ENABLED=0. + * + * GET /api/p2p/config { ok, enabled, staleDays } + * POST /api/p2p/device { fingerprint?, profile? } → { ok, deviceId, secret } + * POST /api/p2p/holdings (device) { items:[{cid,size,videoId}], share } → { ok, accepted, unknown, challenges } + * POST /api/p2p/challenge (device) { answers:[{cid,offset,length,sha256}] } → { ok, passed, failed } + * GET /api/p2p/holders?v=|cid= availability — added by plan 014 + * + * Device auth: header `X-Device: .`; only sha256(secret) is + * stored. A holdings report is always the device's FULL list: anything it held + * before and no longer lists is marked removed. Holder rows never expire by + * time — last_verified_at is refreshed by each report (see the architecture doc). + * ========================================================================== */ +import { createHash, randomBytes, timingSafeEqual } from 'node:crypto'; + +const CID_RE = /^[0-9a-f]{64}$/; +const DEV_RE = /^dev_[0-9a-f]{16}$/; +const MAX_ITEMS = 5000; +const MAX_CHALLENGES = 5; +const CHALLENGE_LEN = 64 * 1024; + +const sha = (s) => createHash('sha256').update(String(s)).digest('hex'); +const same = (a, b) => { const x = Buffer.from(String(a)), y = Buffer.from(String(b)); return x.length === y.length && timingSafeEqual(x, y); }; +export const peerIdOf = (deviceId) => sha('peer:' + deviceId).slice(0, 12); + +export function registerP2pRoutes(app, deps) { + const { cfg, p2pDb, fileForCid, sha256Range, now = () => Date.now(), log = console } = deps; + const pending = new Map(); // `${deviceId}|${cid}` -> { offset, length, at } + const regLog = new Map(); // ip -> [ms] + + const gate = async (c, next) => { + if (!cfg.enabled) return c.json({ ok: false, error: 'p2p disabled' }, 404); + await next(); + }; + + async function deviceOf(c) { + const m = String(c.req.header('x-device') || '').match(/^(dev_[0-9a-f]{16})\.([0-9a-f]{64})$/); + if (!m) return null; + const d = await p2pDb.getDevice(m[1]); + if (!d || !same(d.secret_hash, sha(m[2]))) return null; + return d; + } + const requireDevice = async (c, next) => { + const d = await deviceOf(c); + if (!d) return c.json({ ok: false, error: 'unknown device' }, 401); + c.set('device', d); + await next(); + }; + + app.get('/api/p2p/config', (c) => c.json({ ok: true, enabled: cfg.enabled, staleDays: cfg.staleDays })); + + app.post('/api/p2p/device', gate, async (c) => { + const ip = (c.req.header('x-forwarded-for') || '').split(',')[0].trim() || 'local'; + const t = now(); + const recent = (regLog.get(ip) || []).filter((x) => t - x < 3600_000); + if (recent.length >= 20) return c.json({ ok: false, error: 'too many registrations' }, 429); + recent.push(t); + regLog.set(ip, recent); + if (regLog.size > 10000) regLog.clear(); + const body = await c.req.json().catch(() => ({})); + const deviceId = 'dev_' + randomBytes(8).toString('hex'); + const secret = randomBytes(32).toString('hex'); + await p2pDb.createDevice({ + deviceId, secretHash: sha(secret), + fingerprint: String(body.fingerprint || '').slice(0, 128) || null, + profile: String(body.profile || '').slice(0, 64) || null, + now: t, + }); + return c.json({ ok: true, deviceId, secret }); + }); + + app.post('/api/p2p/holdings', gate, requireDevice, async (c) => { + const d = c.get('device'); + const body = await c.req.json().catch(() => ({})); + const t = now(); + const share = body.share !== false; + await p2pDb.touchDevice(d.device_id, { now: t, share, profile: String(body.profile || '').slice(0, 64) || undefined }); + const raw = Array.isArray(body.items) ? body.items.slice(0, MAX_ITEMS) : []; + const items = share ? raw.filter((x) => x && CID_RE.test(String(x.cid))) : []; + const known = await p2pDb.knownCids(items.map((x) => x.cid)); + const accepted = []; + const unknown = []; + for (const it of items) { + if (!known.has(it.cid)) { unknown.push(it.cid); continue; } + await p2pDb.upsertHolder({ cid: it.cid, deviceId: d.device_id, now: t }); + accepted.push(it.cid); + } + await p2pDb.removeHoldersExcept({ deviceId: d.device_id, keep: accepted, now: t }); + // Spot-check a few holdings against the server's own copy when it has one. + const challenges = []; + for (const cid of accepted) { + if (challenges.length >= MAX_CHALLENGES) break; + const f = await fileForCid(cid); + if (!f || !(f.size > CHALLENGE_LEN)) continue; + const offset = Math.floor(Math.random() * (f.size - CHALLENGE_LEN)); + pending.set(d.device_id + '|' + cid, { offset, length: CHALLENGE_LEN, at: t }); + challenges.push({ cid, offset, length: CHALLENGE_LEN }); + } + if (pending.size > 50000) pending.clear(); + return c.json({ ok: true, accepted, unknown, challenges }); + }); + + app.post('/api/p2p/challenge', gate, requireDevice, async (c) => { + const d = c.get('device'); + const body = await c.req.json().catch(() => ({})); + const t = now(); + const passed = []; + const failed = []; + for (const a of (Array.isArray(body.answers) ? body.answers : []).slice(0, MAX_CHALLENGES)) { + const key = d.device_id + '|' + a.cid; + const p = pending.get(key); + if (!p || p.offset !== a.offset || p.length !== a.length || t - p.at > 10 * 60_000) continue; + pending.delete(key); + const f = await fileForCid(a.cid); + if (!f) continue; + const want = await sha256Range(f.path, p.offset, p.length); + if (String(a.sha256) === want) { + await p2pDb.setHolderTrust({ cid: a.cid, deviceId: d.device_id, trust: 'challenged', now: t }); + passed.push(a.cid); + } else { + await p2pDb.removeHolder({ cid: a.cid, deviceId: d.device_id, now: t }); + failed.push(a.cid); + log.warn?.(`[p2p] ${d.device_id} failed challenge for ${a.cid.slice(0, 12)}`); + } + } + return c.json({ ok: true, passed, failed }); + }); + + return { deviceOf, peerIdOf, requireDevice, gate }; +} diff --git a/server/p2p-routes.test.js b/server/p2p-routes.test.js new file mode 100644 index 0000000..0dafb68 --- /dev/null +++ b/server/p2p-routes.test.js @@ -0,0 +1,111 @@ +// Device registration, holdings reports and range challenges (plan 013). +import { test, expect, beforeAll } from 'bun:test'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createHash } from 'node:crypto'; +import { Hono } from 'hono'; + +const root = mkdtempSync(join(tmpdir(), 'ytp-p2p-routes-')); +process.env.DB_PATH = join(root, 'test.db'); +const dbmod = await import('./db.js'); +const p2pDb = await import('./p2p-db.js'); +const { registerP2pRoutes } = await import('./p2p-routes.js'); +const { sha256Range } = await import('./hash.js'); + +const file = join(root, 'copy.mp4'); +const bytes = Buffer.from(Array.from({ length: 400000 }, (_, i) => (i * 13) % 256)); +writeFileSync(file, bytes); +const CID = createHash('sha256').update(bytes).digest('hex'); +const OTHER = 'c'.repeat(64); // verified but the server has no file +const UNKNOWN = 'd'.repeat(64); // not in p2p_content +const quiet = { warn() {}, info() {} }; +let app; +let enabled = true; + +const req = (path, { method = 'GET', body, dev } = {}) => app.request(path, { + method, + headers: { 'Content-Type': 'application/json', ...(dev ? { 'X-Device': dev.deviceId + '.' + dev.secret } : {}) }, + body: body ? JSON.stringify(body) : undefined, +}); + +beforeAll(async () => { + await dbmod.initDb(); + await p2pDb.initP2pSchema(); + await p2pDb.upsertContent({ cid: CID, videoId: 'dQw4w9WgXcQ', size: bytes.length, origin: 'server', now: 1 }); + await p2pDb.upsertContent({ cid: OTHER, videoId: 'dQw4w9WgXcQ', size: 10, origin: 'server', now: 1 }); + app = new Hono(); + registerP2pRoutes(app, { + cfg: { get enabled() { return enabled; }, staleDays: 7 }, + p2pDb, + fileForCid: async (cid) => (cid === CID ? { path: file, size: bytes.length } : null), + sha256Range, + log: quiet, + }); +}); + +async function newDevice() { + const r = await req('/api/p2p/device', { method: 'POST', body: { fingerprint: 'fp1' } }); + expect(r.status).toBe(200); + const j = await r.json(); + expect(j.deviceId).toMatch(/^dev_[0-9a-f]{16}$/); + expect(j.secret).toMatch(/^[0-9a-f]{64}$/); + return j; +} + +test('config is public and says enabled + staleDays', async () => { + expect(await (await req('/api/p2p/config')).json()).toEqual({ ok: true, enabled: true, staleDays: 7 }); +}); + +test('holdings need a valid device secret', async () => { + const dev = await newDevice(); + expect((await req('/api/p2p/holdings', { method: 'POST', body: { items: [] } })).status).toBe(401); + expect((await req('/api/p2p/holdings', { method: 'POST', body: { items: [] }, dev: { ...dev, secret: 'e'.repeat(64) } })).status).toBe(401); +}); + +test('report accepts verified cids, returns unknown ones, and challenges the server-held file', async () => { + const dev = await newDevice(); + const r = await (await req('/api/p2p/holdings', { method: 'POST', dev, body: { share: true, items: [ + { cid: CID, size: bytes.length, videoId: 'dQw4w9WgXcQ' }, { cid: OTHER, size: 10 }, { cid: UNKNOWN, size: 5 }, { cid: 'nothex' }, + ] } })).json(); + expect(r.accepted.sort()).toEqual([CID, OTHER].sort()); + expect(r.unknown).toEqual([UNKNOWN]); + expect(r.challenges.length).toBe(1); + const ch = r.challenges[0]; + expect(ch.cid).toBe(CID); + const good = createHash('sha256').update(bytes.subarray(ch.offset, ch.offset + ch.length)).digest('hex'); + const a = await (await req('/api/p2p/challenge', { method: 'POST', dev, body: { answers: [{ ...ch, sha256: good }] } })).json(); + expect(a).toEqual({ ok: true, passed: [CID], failed: [] }); + const hs = await p2pDb.listHolders(CID); + expect(hs.find((h) => h.device_id === dev.deviceId).trust).toBe('challenged'); +}); + +test('a wrong challenge answer removes the holder; a replayed answer is ignored', async () => { + const dev = await newDevice(); + const r = await (await req('/api/p2p/holdings', { method: 'POST', dev, body: { items: [{ cid: CID, size: bytes.length }] } })).json(); + const ch = r.challenges[0]; + const a = await (await req('/api/p2p/challenge', { method: 'POST', dev, body: { answers: [{ ...ch, sha256: '0'.repeat(64) }] } })).json(); + expect(a.failed).toEqual([CID]); + expect((await p2pDb.listHolders(CID)).some((h) => h.device_id === dev.deviceId)).toBe(false); + const again = await (await req('/api/p2p/challenge', { method: 'POST', dev, body: { answers: [{ ...ch, sha256: '0'.repeat(64) }] } })).json(); + expect(again).toEqual({ ok: true, passed: [], failed: [] }); +}); + +test('a full report without an item removes it; share=false withdraws everything', async () => { + const dev = await newDevice(); + await req('/api/p2p/holdings', { method: 'POST', dev, body: { items: [{ cid: CID }, { cid: OTHER }] } }); + expect((await p2pDb.activeHoldingsOf(dev.deviceId)).sort()).toEqual([CID, OTHER].sort()); + await req('/api/p2p/holdings', { method: 'POST', dev, body: { items: [{ cid: OTHER }] } }); + expect(await p2pDb.activeHoldingsOf(dev.deviceId)).toEqual([OTHER]); + await req('/api/p2p/holdings', { method: 'POST', dev, body: { share: false, items: [{ cid: OTHER }] } }); + expect(await p2pDb.activeHoldingsOf(dev.deviceId)).toEqual([]); + expect((await p2pDb.getDevice(dev.deviceId)).share).toBe(0); +}); + +test('P2P_ENABLED=0 → routes answer 404 (config still says disabled)', async () => { + enabled = false; + try { + expect((await req('/api/p2p/device', { method: 'POST', body: {} })).status).toBe(404); + expect((await (await req('/api/p2p/config')).json()).enabled).toBe(false); + } finally { enabled = true; } +}); diff --git a/server/package.json b/server/package.json index 854d3ee..15a3bc9 100644 --- a/server/package.json +++ b/server/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "bun server.js", "dev": "bun --hot server.js", - "test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./ytdlp-pool.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js" + "test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./ytdlp-pool.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js" }, "dependencies": { "@hono/node-server": "^1.14.0", diff --git a/server/server.js b/server/server.js index 8aecc1c..7de3e73 100644 --- a/server/server.js +++ b/server/server.js @@ -51,6 +51,8 @@ import { registerUploadRoutes } from './uploads.js'; import { admitFile } from './p2p-admit.js'; import { P2P } from './p2p-config.js'; import * as p2pDb from './p2p-db.js'; +import { registerP2pRoutes } from './p2p-routes.js'; +import { sha256Range } from './hash.js'; import * as innertube from './innertube.js'; import QRCode from 'qrcode'; import { createYtdlpPool } from './ytdlp-pool.js'; @@ -1949,6 +1951,18 @@ const uploads = registerUploadRoutes(app, { }); const isUpload = (id) => uploads.isUploadId(id); +// ============================================================================ +// Peer-to-peer sharing — devices + holdings (docs/p2p-architecture.md) +// ============================================================================ +async function fileForCid(cid) { + const row = await p2pDb.findMediaByCid(cid); + if (!row) return null; + const path = `${MEDIA_DIR}/${row.video_id}.${row.gen}.mp4`; + const f = Bun.file(path); + return (await f.exists()) ? { path, size: f.size } : null; +} +const p2p = registerP2pRoutes(app, { cfg: P2P, p2pDb, fileForCid, sha256Range }); + // ============================================================================ // GET /sw.js — serve the service worker with BUILD_TAG injected //