Register devices and report verified holdings to the server
This commit is contained in:
@@ -353,6 +353,8 @@ function toAssetUrl(path) {
|
||||
// ---------- State ----------
|
||||
const DEFAULT_SETTINGS = {
|
||||
quality: 'auto', volume: 1, audioOnly: false, autoPreload: true,
|
||||
p2pShare: true, // P2P (docs/p2p-architecture.md): share my saved videos with other devices — ON by default
|
||||
p2pReceive: true, // fetch from other devices when YouTube and the server can't — ON by default
|
||||
saveBeforePlay: false, // download (server-muxed single file) before playing instead of streaming
|
||||
repeatMode: 'off', // 'off' | 'all' — repeat the playing list when it ends
|
||||
loopOne: false, // repeat the single current video
|
||||
@@ -9874,6 +9876,10 @@ async function boot() {
|
||||
// Learn what's already cached, then top up any playlist videos that aren't.
|
||||
await refreshCachedIds();
|
||||
startCacheResyncWatch();
|
||||
// Peer-to-peer: report what this device holds (on by default; settings.p2pShare).
|
||||
if (WEB && window.P2PClient) {
|
||||
window.P2PClient.start({ getSettings: () => data.settings, getProfile: () => (data.profile && data.profile.name) || '' });
|
||||
}
|
||||
data.playlists.forEach(preloadPlaylist);
|
||||
preloadPinnedPlaylists();
|
||||
// Not awaited — artwork backfill must never delay first paint.
|
||||
|
||||
26
frontend/hash-worker.js
Normal file
26
frontend/hash-worker.js
Normal file
@@ -0,0 +1,26 @@
|
||||
/* ============================================================================
|
||||
* hash-worker.js — SHA-256 of a file already saved in OPFS, off the main
|
||||
* thread, read in 4 MiB slices (never the whole video in memory).
|
||||
*
|
||||
* In: { name } file name under OPFS videos/ (e.g. "abc.mp4")
|
||||
* Out: { ok: true, sha256, size } | { ok: false, error }
|
||||
* ========================================================================== */
|
||||
'use strict';
|
||||
importScripts('/sha256.js');
|
||||
|
||||
self.onmessage = async (e) => {
|
||||
const { name } = e.data || {};
|
||||
try {
|
||||
const root = await navigator.storage.getDirectory();
|
||||
const dir = await root.getDirectoryHandle('videos');
|
||||
const file = await (await dir.getFileHandle(String(name))).getFile();
|
||||
const h = self.Sha256.create();
|
||||
const STEP = 4 * 1024 * 1024;
|
||||
for (let pos = 0; pos < file.size; pos += STEP) {
|
||||
h.update(new Uint8Array(await file.slice(pos, pos + STEP).arrayBuffer()));
|
||||
}
|
||||
self.postMessage({ ok: true, sha256: h.hex(), size: file.size });
|
||||
} catch (err) {
|
||||
self.postMessage({ ok: false, error: err && err.message ? err.message : String(err) });
|
||||
}
|
||||
};
|
||||
@@ -563,6 +563,7 @@
|
||||
<script src="stats-core.js"></script>
|
||||
<script src="sha256.js"></script>
|
||||
<script src="device-db.js"></script>
|
||||
<script src="p2p-client.js"></script>
|
||||
<script src="async-guard.js"></script>
|
||||
<script src="sw-update.js"></script>
|
||||
<script src="app.js"></script>
|
||||
|
||||
@@ -108,6 +108,19 @@
|
||||
}
|
||||
},
|
||||
|
||||
// Bytes [offset, offset+length) of a saved video — answers the server's
|
||||
// P2P range challenges (p2p-client.js). null when the file is missing.
|
||||
async readRange(videoId, offset, length) {
|
||||
try {
|
||||
const found = await findHandle(videoId);
|
||||
if (!found) return null;
|
||||
const file = await found[0].getFile();
|
||||
return new Uint8Array(await file.slice(offset, offset + length).arrayBuffer());
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
|
||||
revokeUrl(url) {
|
||||
if (url && _blobUrls.has(url)) {
|
||||
URL.revokeObjectURL(url);
|
||||
|
||||
189
frontend/p2p-client.js
Normal file
189
frontend/p2p-client.js
Normal file
@@ -0,0 +1,189 @@
|
||||
/* ============================================================================
|
||||
* p2p-client.js — this device's side of peer-to-peer sharing
|
||||
* (docs/p2p-architecture.md flows 2–3). window.P2PClient.
|
||||
*
|
||||
* start({ getSettings, getProfile }) called once from app.js boot()
|
||||
* changed() a save/delete happened → re-report soon
|
||||
* device() { deviceId, secret } or null
|
||||
* authHeaders() { 'X-Device': … } for other P2P calls
|
||||
*
|
||||
* What it does, in order, each sync:
|
||||
* 1. registers the device once (localStorage ytpDevice)
|
||||
* 2. reconciles DeviceDB with the files really in OPFS (a record without a
|
||||
* file is dropped; a file without a record is added as 'unhashed')
|
||||
* 3. hashes 'unhashed' files and files not re-checked for 30 days, one at a
|
||||
* time in hash-worker.js
|
||||
* 4. reports the FULL holdings list (empty when sharing is off), answers the
|
||||
* server's range challenges, and marks accepted files 'verified'
|
||||
* P2P is ON by default: sharing runs unless settings.p2pShare === false or the
|
||||
* server says P2P is disabled.
|
||||
* ========================================================================== */
|
||||
(function () {
|
||||
'use strict';
|
||||
|
||||
const KEY = 'ytpDevice';
|
||||
const REHASH_MS = 30 * 24 * 3600_000;
|
||||
const RESYNC_MS = 6 * 3600_000;
|
||||
let hooks = { getSettings: () => ({}), getProfile: () => '' };
|
||||
let serverCfg = null;
|
||||
let running = null;
|
||||
let again = false;
|
||||
let timer = null;
|
||||
let lastSync = 0;
|
||||
|
||||
function device() {
|
||||
try {
|
||||
const d = JSON.parse(localStorage.getItem(KEY) || 'null');
|
||||
return d && /^dev_[0-9a-f]{16}$/.test(d.deviceId) && /^[0-9a-f]{64}$/.test(d.secret) ? d : null;
|
||||
} catch { return null; }
|
||||
}
|
||||
const authHeaders = () => { const d = device(); return d ? { 'X-Device': d.deviceId + '.' + d.secret } : {}; };
|
||||
|
||||
async function config() {
|
||||
if (serverCfg) return serverCfg;
|
||||
try {
|
||||
const j = await (await fetch('/api/p2p/config')).json();
|
||||
if (j && j.ok) serverCfg = j;
|
||||
} catch { /* offline — try again next sync */ }
|
||||
return serverCfg;
|
||||
}
|
||||
|
||||
async function ensureDevice() {
|
||||
const have = device();
|
||||
if (have) return have;
|
||||
const r = await fetch('/api/p2p/device', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ fingerprint: window.getFingerprint ? window.getFingerprint() : '', profile: hooks.getProfile() || '' }),
|
||||
});
|
||||
const j = await r.json();
|
||||
if (!j || !j.ok) throw new Error((j && j.error) || 'device registration failed');
|
||||
const d = { deviceId: j.deviceId, secret: j.secret };
|
||||
try { localStorage.setItem(KEY, JSON.stringify(d)); } catch { /* storage blocked */ }
|
||||
return d;
|
||||
}
|
||||
|
||||
function hashInWorker(name) {
|
||||
return new Promise((resolve) => {
|
||||
let w;
|
||||
try { w = new Worker('/hash-worker.js'); } catch { resolve(null); return; }
|
||||
w.onmessage = (e) => { w.terminate(); resolve(e.data && e.data.ok ? e.data : null); };
|
||||
w.onerror = () => { w.terminate(); resolve(null); };
|
||||
w.postMessage({ name });
|
||||
});
|
||||
}
|
||||
|
||||
async function sha256Range(videoId, offset, length) {
|
||||
const bytes = window.OPFS && window.OPFS.readRange ? await window.OPFS.readRange(videoId, offset, length) : null;
|
||||
if (!bytes) return null;
|
||||
return window.Sha256.hex(bytes);
|
||||
}
|
||||
|
||||
// DeviceDB ⇄ OPFS. Returns the records that describe a real file.
|
||||
async function reconcile() {
|
||||
const files = await window.OPFS.listVideos({ strict: true });
|
||||
const byId = new Map(files.map((f) => [f.id, f]));
|
||||
const recs = await window.DeviceDB.listFiles();
|
||||
const out = [];
|
||||
for (const r of recs) {
|
||||
const f = byId.get(r.videoId);
|
||||
if (!f) { await window.DeviceDB.deleteFile(r.videoId); continue; }
|
||||
if (f.size !== r.size && r.size) { r.cid = null; r.state = 'unhashed'; r.size = f.size; await window.DeviceDB.putFile(r); }
|
||||
r.name = f.name;
|
||||
out.push(r);
|
||||
byId.delete(r.videoId);
|
||||
}
|
||||
for (const f of byId.values()) {
|
||||
if (String(f.id).startsWith('edit_')) continue; // edited cuts are never shared
|
||||
const r = { videoId: f.id, cid: null, size: f.size, savedAt: Date.now(), lastCheckedAt: 0, state: 'unhashed' };
|
||||
await window.DeviceDB.putFile(r);
|
||||
out.push({ ...r, name: f.name });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function hashPending(recs) {
|
||||
const now = Date.now();
|
||||
for (const r of recs) {
|
||||
if (r.state !== 'unhashed' && now - (r.lastCheckedAt || 0) < REHASH_MS) continue;
|
||||
const h = await hashInWorker(r.name);
|
||||
if (!h) continue;
|
||||
const changedCid = h.sha256 !== r.cid;
|
||||
r.cid = h.sha256;
|
||||
r.size = h.size;
|
||||
r.lastCheckedAt = Date.now();
|
||||
if (changedCid || r.state === 'unhashed') r.state = 'unverified';
|
||||
await window.DeviceDB.putFile(r);
|
||||
}
|
||||
}
|
||||
|
||||
async function report(recs, dev) {
|
||||
const share = hooks.getSettings().p2pShare !== false;
|
||||
const items = recs.filter((r) => r.cid).map((r) => ({ cid: r.cid, size: r.size, videoId: r.videoId }));
|
||||
const r = await fetch('/api/p2p/holdings', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-Device': dev.deviceId + '.' + dev.secret },
|
||||
body: JSON.stringify({ share, items: share ? items : [], profile: hooks.getProfile() || '' }),
|
||||
});
|
||||
if (r.status === 401) { try { localStorage.removeItem(KEY); } catch { /* ignore */ } return null; }
|
||||
const j = await r.json();
|
||||
if (!j || !j.ok) return null;
|
||||
const accepted = new Set(j.accepted || []);
|
||||
for (const rec of recs) {
|
||||
if (rec.cid && accepted.has(rec.cid) && rec.state !== 'verified') { rec.state = 'verified'; await window.DeviceDB.putFile(rec); }
|
||||
}
|
||||
const answers = [];
|
||||
for (const ch of j.challenges || []) {
|
||||
const rec = recs.find((x) => x.cid === ch.cid);
|
||||
if (!rec) continue;
|
||||
const hex = await sha256Range(rec.videoId, ch.offset, ch.length);
|
||||
if (hex) answers.push({ ...ch, sha256: hex });
|
||||
}
|
||||
if (answers.length) {
|
||||
await fetch('/api/p2p/challenge', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-Device': dev.deviceId + '.' + dev.secret },
|
||||
body: JSON.stringify({ answers }),
|
||||
}).catch(() => {});
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
async function syncOnce() {
|
||||
if (!window.OPFS || !window.OPFS.isSupported() || !window.DeviceDB || !window.Sha256) return null;
|
||||
if (navigator.onLine === false) return null;
|
||||
const cfg = await config();
|
||||
if (!cfg || !cfg.enabled) return null;
|
||||
const dev = await ensureDevice();
|
||||
const recs = await reconcile();
|
||||
await hashPending(recs);
|
||||
const res = await report(recs, dev);
|
||||
lastSync = Date.now();
|
||||
return res;
|
||||
}
|
||||
|
||||
// One sync at a time; a request during a sync schedules exactly one more.
|
||||
function sync() {
|
||||
if (running) { again = true; return running; }
|
||||
running = syncOnce().catch(() => null).finally(() => {
|
||||
running = null;
|
||||
if (again) { again = false; sync(); }
|
||||
});
|
||||
return running;
|
||||
}
|
||||
|
||||
function changed() {
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(sync, 5000);
|
||||
}
|
||||
|
||||
function start(h) {
|
||||
hooks = { ...hooks, ...(h || {}) };
|
||||
const idle = window.requestIdleCallback || ((fn) => setTimeout(fn, 1));
|
||||
setTimeout(() => idle(() => sync()), 8000);
|
||||
document.addEventListener('visibilitychange', () => {
|
||||
if (document.visibilityState === 'visible' && Date.now() - lastSync > RESYNC_MS) sync();
|
||||
});
|
||||
}
|
||||
|
||||
window.P2PClient = { start, changed, sync, device, authHeaders, config };
|
||||
}());
|
||||
@@ -66,6 +66,8 @@ const SHELL = [
|
||||
'/stats-core.js',
|
||||
'/sha256.js',
|
||||
'/device-db.js',
|
||||
'/hash-worker.js',
|
||||
'/p2p-client.js',
|
||||
'/app.js',
|
||||
'/manifest.webmanifest',
|
||||
'/icons/icon-192.png',
|
||||
|
||||
@@ -20,7 +20,7 @@ green, app boots with no JS errors, P2P on by default, offline boot works).
|
||||
| 010 | 010-views-and-retention-d0c6ca | Count views and evict server copies by retention criteria before LRU | done | Count views and evict server copies by retention criteria before LRU | |
|
||||
| 011 | 011-browser-sha256-e1793d | Add an incremental SHA-256 library for the browser and node tests | done | Add an incremental SHA-256 library for the browser and node tests | |
|
||||
| 012 | 012-device-file-registry-288d55 | Add the on-device IndexedDB file registry and hash saves while downloading | done | Add the on-device IndexedDB file registry and hash saves while downloading | browser harness |
|
||||
| 013 | 013-device-identity-and-holdings-3ba493 | Register devices and report verified holdings to the server | in-progress | | persistent holders, no TTL |
|
||||
| 013 | 013-device-identity-and-holdings-3ba493 | Register devices and report verified holdings to the server | done | Register devices and report verified holdings to the server | persistent holders, no TTL |
|
||||
| 014 | 014-p2p-presence-hub-ceced8 | Add the /ws/p2p presence and signalling hub and the holders endpoint | queued | | stale flag, never hidden |
|
||||
| 015 | 015-availability-ui-and-settings-3b9397 | Show peer availability with stale markers and add Sharing settings | queued | | |
|
||||
| 016 | 016-intake-and-server-verification-cfe031 | Let a device hand a file to the server for hashing and validation | queued | | needs ffmpeg for tests |
|
||||
|
||||
@@ -620,3 +620,9 @@ self.onmessage = async (e) => {
|
||||
window.P2PClient = { start, changed, sync, device, authHeaders, config };
|
||||
}());
|
||||
```
|
||||
|
||||
## Execution log
|
||||
|
||||
- Executor: in-session Agent (haiku). Attempts: 1. Fix rounds: 0.
|
||||
- Orchestrator re-ran Verification: `p2p-routes.js`, `p2p-routes.test.js`, `hash-worker.js`, `p2p-client.js`, `opfs.js` byte-identical to the pre-tested versions; routes tests 6 pass; all 12 server test files 0 fail; `SERVER_OK`; frontend syntax ok; 56 frontend tests pass; browser check in Chromium reproduced the expected JSON (accepted 1, unknown 1, challenge passed -> trust `challenged`, share off -> 0 holders, device registered). No leftover processes.
|
||||
- Executor Findings (verbatim): All 12 steps completed successfully. P2P routes pass 6/6 tests. Server builds without errors. Frontend syntax check passes. All 56 frontend unit tests pass. Browser check returns expected JSON with device registration, holdings acceptance, unknown detection, challenge completion, and share-off withdrawal. git apply patch for OPFS.readRange applied cleanly. New files created: p2p-routes.js, p2p-routes.test.js, hash-worker.js, p2p-client.js match appendix specifications exactly.
|
||||
@@ -63,6 +63,7 @@ export async function initP2pSchema() {
|
||||
// media_cache.sha256 — the cid of the current <id>.<gen>.mp4 (plan 009).
|
||||
try { await db.execute('ALTER TABLE media_cache ADD COLUMN sha256 TEXT'); }
|
||||
catch (e) { if (!/duplicate column/i.test(String(e.message))) throw e; }
|
||||
await db.execute('CREATE INDEX IF NOT EXISTS idx_media_sha256 ON media_cache (sha256)');
|
||||
}
|
||||
|
||||
const rowsOf = (r) => r.rows.map((row) => {
|
||||
@@ -258,3 +259,12 @@ export async function listMediaEvictionOrder({ now, keepMinViews, keepDays, keep
|
||||
|| (a.last_access - b.last_access));
|
||||
return rows;
|
||||
}
|
||||
|
||||
// The server's own ready copy whose mp4 has this content id (or null).
|
||||
export async function findMediaByCid(cid) {
|
||||
const r = await db.execute({
|
||||
sql: "SELECT video_id, gen FROM media_cache WHERE sha256 = ? AND status = 'ready' LIMIT 1",
|
||||
args: [cid],
|
||||
});
|
||||
return rowsOf(r)[0] || null;
|
||||
}
|
||||
|
||||
133
server/p2p-routes.js
Normal file
133
server/p2p-routes.js
Normal file
@@ -0,0 +1,133 @@
|
||||
/* ============================================================================
|
||||
* p2p-routes.js — device registration + holdings (docs/p2p-architecture.md
|
||||
* flows 3 and 5). Mounted by server.js; every route answers 404 when
|
||||
* P2P_ENABLED=0.
|
||||
*
|
||||
* GET /api/p2p/config { ok, enabled, staleDays }
|
||||
* POST /api/p2p/device { fingerprint?, profile? } → { ok, deviceId, secret }
|
||||
* POST /api/p2p/holdings (device) { items:[{cid,size,videoId}], share } → { ok, accepted, unknown, challenges }
|
||||
* POST /api/p2p/challenge (device) { answers:[{cid,offset,length,sha256}] } → { ok, passed, failed }
|
||||
* GET /api/p2p/holders?v=<id>|cid= availability — added by plan 014
|
||||
*
|
||||
* Device auth: header `X-Device: <deviceId>.<secret>`; only sha256(secret) is
|
||||
* stored. A holdings report is always the device's FULL list: anything it held
|
||||
* before and no longer lists is marked removed. Holder rows never expire by
|
||||
* time — last_verified_at is refreshed by each report (see the architecture doc).
|
||||
* ========================================================================== */
|
||||
import { createHash, randomBytes, timingSafeEqual } from 'node:crypto';
|
||||
|
||||
const CID_RE = /^[0-9a-f]{64}$/;
|
||||
const DEV_RE = /^dev_[0-9a-f]{16}$/;
|
||||
const MAX_ITEMS = 5000;
|
||||
const MAX_CHALLENGES = 5;
|
||||
const CHALLENGE_LEN = 64 * 1024;
|
||||
|
||||
const sha = (s) => createHash('sha256').update(String(s)).digest('hex');
|
||||
const same = (a, b) => { const x = Buffer.from(String(a)), y = Buffer.from(String(b)); return x.length === y.length && timingSafeEqual(x, y); };
|
||||
export const peerIdOf = (deviceId) => sha('peer:' + deviceId).slice(0, 12);
|
||||
|
||||
export function registerP2pRoutes(app, deps) {
|
||||
const { cfg, p2pDb, fileForCid, sha256Range, now = () => Date.now(), log = console } = deps;
|
||||
const pending = new Map(); // `${deviceId}|${cid}` -> { offset, length, at }
|
||||
const regLog = new Map(); // ip -> [ms]
|
||||
|
||||
const gate = async (c, next) => {
|
||||
if (!cfg.enabled) return c.json({ ok: false, error: 'p2p disabled' }, 404);
|
||||
await next();
|
||||
};
|
||||
|
||||
async function deviceOf(c) {
|
||||
const m = String(c.req.header('x-device') || '').match(/^(dev_[0-9a-f]{16})\.([0-9a-f]{64})$/);
|
||||
if (!m) return null;
|
||||
const d = await p2pDb.getDevice(m[1]);
|
||||
if (!d || !same(d.secret_hash, sha(m[2]))) return null;
|
||||
return d;
|
||||
}
|
||||
const requireDevice = async (c, next) => {
|
||||
const d = await deviceOf(c);
|
||||
if (!d) return c.json({ ok: false, error: 'unknown device' }, 401);
|
||||
c.set('device', d);
|
||||
await next();
|
||||
};
|
||||
|
||||
app.get('/api/p2p/config', (c) => c.json({ ok: true, enabled: cfg.enabled, staleDays: cfg.staleDays }));
|
||||
|
||||
app.post('/api/p2p/device', gate, async (c) => {
|
||||
const ip = (c.req.header('x-forwarded-for') || '').split(',')[0].trim() || 'local';
|
||||
const t = now();
|
||||
const recent = (regLog.get(ip) || []).filter((x) => t - x < 3600_000);
|
||||
if (recent.length >= 20) return c.json({ ok: false, error: 'too many registrations' }, 429);
|
||||
recent.push(t);
|
||||
regLog.set(ip, recent);
|
||||
if (regLog.size > 10000) regLog.clear();
|
||||
const body = await c.req.json().catch(() => ({}));
|
||||
const deviceId = 'dev_' + randomBytes(8).toString('hex');
|
||||
const secret = randomBytes(32).toString('hex');
|
||||
await p2pDb.createDevice({
|
||||
deviceId, secretHash: sha(secret),
|
||||
fingerprint: String(body.fingerprint || '').slice(0, 128) || null,
|
||||
profile: String(body.profile || '').slice(0, 64) || null,
|
||||
now: t,
|
||||
});
|
||||
return c.json({ ok: true, deviceId, secret });
|
||||
});
|
||||
|
||||
app.post('/api/p2p/holdings', gate, requireDevice, async (c) => {
|
||||
const d = c.get('device');
|
||||
const body = await c.req.json().catch(() => ({}));
|
||||
const t = now();
|
||||
const share = body.share !== false;
|
||||
await p2pDb.touchDevice(d.device_id, { now: t, share, profile: String(body.profile || '').slice(0, 64) || undefined });
|
||||
const raw = Array.isArray(body.items) ? body.items.slice(0, MAX_ITEMS) : [];
|
||||
const items = share ? raw.filter((x) => x && CID_RE.test(String(x.cid))) : [];
|
||||
const known = await p2pDb.knownCids(items.map((x) => x.cid));
|
||||
const accepted = [];
|
||||
const unknown = [];
|
||||
for (const it of items) {
|
||||
if (!known.has(it.cid)) { unknown.push(it.cid); continue; }
|
||||
await p2pDb.upsertHolder({ cid: it.cid, deviceId: d.device_id, now: t });
|
||||
accepted.push(it.cid);
|
||||
}
|
||||
await p2pDb.removeHoldersExcept({ deviceId: d.device_id, keep: accepted, now: t });
|
||||
// Spot-check a few holdings against the server's own copy when it has one.
|
||||
const challenges = [];
|
||||
for (const cid of accepted) {
|
||||
if (challenges.length >= MAX_CHALLENGES) break;
|
||||
const f = await fileForCid(cid);
|
||||
if (!f || !(f.size > CHALLENGE_LEN)) continue;
|
||||
const offset = Math.floor(Math.random() * (f.size - CHALLENGE_LEN));
|
||||
pending.set(d.device_id + '|' + cid, { offset, length: CHALLENGE_LEN, at: t });
|
||||
challenges.push({ cid, offset, length: CHALLENGE_LEN });
|
||||
}
|
||||
if (pending.size > 50000) pending.clear();
|
||||
return c.json({ ok: true, accepted, unknown, challenges });
|
||||
});
|
||||
|
||||
app.post('/api/p2p/challenge', gate, requireDevice, async (c) => {
|
||||
const d = c.get('device');
|
||||
const body = await c.req.json().catch(() => ({}));
|
||||
const t = now();
|
||||
const passed = [];
|
||||
const failed = [];
|
||||
for (const a of (Array.isArray(body.answers) ? body.answers : []).slice(0, MAX_CHALLENGES)) {
|
||||
const key = d.device_id + '|' + a.cid;
|
||||
const p = pending.get(key);
|
||||
if (!p || p.offset !== a.offset || p.length !== a.length || t - p.at > 10 * 60_000) continue;
|
||||
pending.delete(key);
|
||||
const f = await fileForCid(a.cid);
|
||||
if (!f) continue;
|
||||
const want = await sha256Range(f.path, p.offset, p.length);
|
||||
if (String(a.sha256) === want) {
|
||||
await p2pDb.setHolderTrust({ cid: a.cid, deviceId: d.device_id, trust: 'challenged', now: t });
|
||||
passed.push(a.cid);
|
||||
} else {
|
||||
await p2pDb.removeHolder({ cid: a.cid, deviceId: d.device_id, now: t });
|
||||
failed.push(a.cid);
|
||||
log.warn?.(`[p2p] ${d.device_id} failed challenge for ${a.cid.slice(0, 12)}`);
|
||||
}
|
||||
}
|
||||
return c.json({ ok: true, passed, failed });
|
||||
});
|
||||
|
||||
return { deviceOf, peerIdOf, requireDevice, gate };
|
||||
}
|
||||
111
server/p2p-routes.test.js
Normal file
111
server/p2p-routes.test.js
Normal file
@@ -0,0 +1,111 @@
|
||||
// Device registration, holdings reports and range challenges (plan 013).
|
||||
import { test, expect, beforeAll } from 'bun:test';
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { Hono } from 'hono';
|
||||
|
||||
const root = mkdtempSync(join(tmpdir(), 'ytp-p2p-routes-'));
|
||||
process.env.DB_PATH = join(root, 'test.db');
|
||||
const dbmod = await import('./db.js');
|
||||
const p2pDb = await import('./p2p-db.js');
|
||||
const { registerP2pRoutes } = await import('./p2p-routes.js');
|
||||
const { sha256Range } = await import('./hash.js');
|
||||
|
||||
const file = join(root, 'copy.mp4');
|
||||
const bytes = Buffer.from(Array.from({ length: 400000 }, (_, i) => (i * 13) % 256));
|
||||
writeFileSync(file, bytes);
|
||||
const CID = createHash('sha256').update(bytes).digest('hex');
|
||||
const OTHER = 'c'.repeat(64); // verified but the server has no file
|
||||
const UNKNOWN = 'd'.repeat(64); // not in p2p_content
|
||||
const quiet = { warn() {}, info() {} };
|
||||
let app;
|
||||
let enabled = true;
|
||||
|
||||
const req = (path, { method = 'GET', body, dev } = {}) => app.request(path, {
|
||||
method,
|
||||
headers: { 'Content-Type': 'application/json', ...(dev ? { 'X-Device': dev.deviceId + '.' + dev.secret } : {}) },
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
await dbmod.initDb();
|
||||
await p2pDb.initP2pSchema();
|
||||
await p2pDb.upsertContent({ cid: CID, videoId: 'dQw4w9WgXcQ', size: bytes.length, origin: 'server', now: 1 });
|
||||
await p2pDb.upsertContent({ cid: OTHER, videoId: 'dQw4w9WgXcQ', size: 10, origin: 'server', now: 1 });
|
||||
app = new Hono();
|
||||
registerP2pRoutes(app, {
|
||||
cfg: { get enabled() { return enabled; }, staleDays: 7 },
|
||||
p2pDb,
|
||||
fileForCid: async (cid) => (cid === CID ? { path: file, size: bytes.length } : null),
|
||||
sha256Range,
|
||||
log: quiet,
|
||||
});
|
||||
});
|
||||
|
||||
async function newDevice() {
|
||||
const r = await req('/api/p2p/device', { method: 'POST', body: { fingerprint: 'fp1' } });
|
||||
expect(r.status).toBe(200);
|
||||
const j = await r.json();
|
||||
expect(j.deviceId).toMatch(/^dev_[0-9a-f]{16}$/);
|
||||
expect(j.secret).toMatch(/^[0-9a-f]{64}$/);
|
||||
return j;
|
||||
}
|
||||
|
||||
test('config is public and says enabled + staleDays', async () => {
|
||||
expect(await (await req('/api/p2p/config')).json()).toEqual({ ok: true, enabled: true, staleDays: 7 });
|
||||
});
|
||||
|
||||
test('holdings need a valid device secret', async () => {
|
||||
const dev = await newDevice();
|
||||
expect((await req('/api/p2p/holdings', { method: 'POST', body: { items: [] } })).status).toBe(401);
|
||||
expect((await req('/api/p2p/holdings', { method: 'POST', body: { items: [] }, dev: { ...dev, secret: 'e'.repeat(64) } })).status).toBe(401);
|
||||
});
|
||||
|
||||
test('report accepts verified cids, returns unknown ones, and challenges the server-held file', async () => {
|
||||
const dev = await newDevice();
|
||||
const r = await (await req('/api/p2p/holdings', { method: 'POST', dev, body: { share: true, items: [
|
||||
{ cid: CID, size: bytes.length, videoId: 'dQw4w9WgXcQ' }, { cid: OTHER, size: 10 }, { cid: UNKNOWN, size: 5 }, { cid: 'nothex' },
|
||||
] } })).json();
|
||||
expect(r.accepted.sort()).toEqual([CID, OTHER].sort());
|
||||
expect(r.unknown).toEqual([UNKNOWN]);
|
||||
expect(r.challenges.length).toBe(1);
|
||||
const ch = r.challenges[0];
|
||||
expect(ch.cid).toBe(CID);
|
||||
const good = createHash('sha256').update(bytes.subarray(ch.offset, ch.offset + ch.length)).digest('hex');
|
||||
const a = await (await req('/api/p2p/challenge', { method: 'POST', dev, body: { answers: [{ ...ch, sha256: good }] } })).json();
|
||||
expect(a).toEqual({ ok: true, passed: [CID], failed: [] });
|
||||
const hs = await p2pDb.listHolders(CID);
|
||||
expect(hs.find((h) => h.device_id === dev.deviceId).trust).toBe('challenged');
|
||||
});
|
||||
|
||||
test('a wrong challenge answer removes the holder; a replayed answer is ignored', async () => {
|
||||
const dev = await newDevice();
|
||||
const r = await (await req('/api/p2p/holdings', { method: 'POST', dev, body: { items: [{ cid: CID, size: bytes.length }] } })).json();
|
||||
const ch = r.challenges[0];
|
||||
const a = await (await req('/api/p2p/challenge', { method: 'POST', dev, body: { answers: [{ ...ch, sha256: '0'.repeat(64) }] } })).json();
|
||||
expect(a.failed).toEqual([CID]);
|
||||
expect((await p2pDb.listHolders(CID)).some((h) => h.device_id === dev.deviceId)).toBe(false);
|
||||
const again = await (await req('/api/p2p/challenge', { method: 'POST', dev, body: { answers: [{ ...ch, sha256: '0'.repeat(64) }] } })).json();
|
||||
expect(again).toEqual({ ok: true, passed: [], failed: [] });
|
||||
});
|
||||
|
||||
test('a full report without an item removes it; share=false withdraws everything', async () => {
|
||||
const dev = await newDevice();
|
||||
await req('/api/p2p/holdings', { method: 'POST', dev, body: { items: [{ cid: CID }, { cid: OTHER }] } });
|
||||
expect((await p2pDb.activeHoldingsOf(dev.deviceId)).sort()).toEqual([CID, OTHER].sort());
|
||||
await req('/api/p2p/holdings', { method: 'POST', dev, body: { items: [{ cid: OTHER }] } });
|
||||
expect(await p2pDb.activeHoldingsOf(dev.deviceId)).toEqual([OTHER]);
|
||||
await req('/api/p2p/holdings', { method: 'POST', dev, body: { share: false, items: [{ cid: OTHER }] } });
|
||||
expect(await p2pDb.activeHoldingsOf(dev.deviceId)).toEqual([]);
|
||||
expect((await p2pDb.getDevice(dev.deviceId)).share).toBe(0);
|
||||
});
|
||||
|
||||
test('P2P_ENABLED=0 → routes answer 404 (config still says disabled)', async () => {
|
||||
enabled = false;
|
||||
try {
|
||||
expect((await req('/api/p2p/device', { method: 'POST', body: {} })).status).toBe(404);
|
||||
expect((await (await req('/api/p2p/config')).json()).enabled).toBe(false);
|
||||
} finally { enabled = true; }
|
||||
});
|
||||
@@ -6,7 +6,7 @@
|
||||
"scripts": {
|
||||
"start": "bun server.js",
|
||||
"dev": "bun --hot server.js",
|
||||
"test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./ytdlp-pool.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js"
|
||||
"test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./ytdlp-pool.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.14.0",
|
||||
|
||||
@@ -51,6 +51,8 @@ import { registerUploadRoutes } from './uploads.js';
|
||||
import { admitFile } from './p2p-admit.js';
|
||||
import { P2P } from './p2p-config.js';
|
||||
import * as p2pDb from './p2p-db.js';
|
||||
import { registerP2pRoutes } from './p2p-routes.js';
|
||||
import { sha256Range } from './hash.js';
|
||||
import * as innertube from './innertube.js';
|
||||
import QRCode from 'qrcode';
|
||||
import { createYtdlpPool } from './ytdlp-pool.js';
|
||||
@@ -1949,6 +1951,18 @@ const uploads = registerUploadRoutes(app, {
|
||||
});
|
||||
const isUpload = (id) => uploads.isUploadId(id);
|
||||
|
||||
// ============================================================================
|
||||
// Peer-to-peer sharing — devices + holdings (docs/p2p-architecture.md)
|
||||
// ============================================================================
|
||||
async function fileForCid(cid) {
|
||||
const row = await p2pDb.findMediaByCid(cid);
|
||||
if (!row) return null;
|
||||
const path = `${MEDIA_DIR}/${row.video_id}.${row.gen}.mp4`;
|
||||
const f = Bun.file(path);
|
||||
return (await f.exists()) ? { path, size: f.size } : null;
|
||||
}
|
||||
const p2p = registerP2pRoutes(app, { cfg: P2P, p2pDb, fileForCid, sha256Range });
|
||||
|
||||
// ============================================================================
|
||||
// GET /sw.js — serve the service worker with BUILD_TAG injected
|
||||
//
|
||||
|
||||
Reference in New Issue
Block a user