From 6af96622dd6f18100e4fddb1c2e212efef828c05 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 07:31:18 +0000 Subject: [PATCH] Add the on-device IndexedDB file registry and hash saves while downloading --- frontend/app.js | 17 ++++- frontend/device-db.js | 76 +++++++++++++++++++ frontend/index.html | 1 + frontend/opfs-worker.js | 22 +++++- frontend/opfs.js | 2 +- frontend/sw.js | 1 + plans/INDEX.md | 2 +- .../012-device-file-registry-288d55.md | 7 ++ 8 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 frontend/device-db.js rename plans/{active => done}/012-device-file-registry-288d55.md (89%) diff --git a/frontend/app.js b/frontend/app.js index a435899..3956cb2 100755 --- a/frontend/app.js +++ b/frontend/app.js @@ -189,7 +189,7 @@ async function opfsDownload(videoId, { mux = false } = {}) { let workerError = null; if (typeof window.OPFS.downloadVideo === 'function' && typeof Worker !== 'undefined') { const w = await window.OPFS.downloadVideo(videoId, url); - if (w.ok) return { ok: true, cached: true }; + if (w.ok) return { ok: true, cached: true, sha256: w.sha256 || null, expectedSha: w.expectedSha || null, size: w.size || 0 }; workerError = w.error || null; } @@ -270,12 +270,14 @@ async function opfsList() { async function opfsDelete(videoId) { if (!window.OPFS || !window.OPFS.isSupported()) return { ok: true }; try { await window.OPFS.deleteVideo(videoId); } catch { /* ignore */ } + if (window.DeviceDB) { await window.DeviceDB.deleteFile(videoId); if (window.P2PClient) window.P2PClient.changed(); } return { ok: true }; } async function opfsClear() { if (!window.OPFS || !window.OPFS.isSupported()) return { ok: true }; try { await window.OPFS.clearAll(); } catch { /* ignore */ } + if (window.DeviceDB) { await window.DeviceDB.clear(); if (window.P2PClient) window.P2PClient.changed(); } return { ok: true }; } @@ -1259,6 +1261,18 @@ async function warmOfflineThumbs() { } } +// This device's record of what it holds and each file's content id +// (docs/p2p-architecture.md flow 2). The P2P client reports these. +function recordDeviceFile(id, res) { + if (!WEB || !window.DeviceDB) return; + const cid = res && window.Sha256 && window.Sha256.isHex(res.sha256) ? res.sha256 : null; + const state = cid ? (res.expectedSha === cid ? 'verified' : 'unverified') : 'unhashed'; + const now = Date.now(); + window.DeviceDB.putFile({ videoId: id, cid, size: (res && res.size) || 0, savedAt: now, lastCheckedAt: now, state }) + .then(() => { if (window.P2PClient) window.P2PClient.changed(); }) + .catch(() => {}); +} + // Download a video into the permanent offline cache. Safe to call repeatedly. async function preload(video, { quiet = false, mux = false } = {}) { const id = video.id; @@ -1282,6 +1296,7 @@ async function preload(video, { quiet = false, mux = false } = {}) { if (res && res.ok && res.cached) { cachedIds.add(id); cacheMutations++; + recordDeviceFile(id, res); warmThumb(thumbUrlFor(id, video)); if (!quiet) toast(`Saved “${video.title}” ✓`); } else if (!quiet) { diff --git a/frontend/device-db.js b/frontend/device-db.js new file mode 100644 index 0000000..29d60d6 --- /dev/null +++ b/frontend/device-db.js @@ -0,0 +1,76 @@ +/* ============================================================================ + * device-db.js — this device's own file registry (IndexedDB "ytp-device") + * + * One record per saved video, next to the bytes in OPFS: + * { videoId, cid, size, savedAt, lastCheckedAt, state } + * cid SHA-256 of the stored file (P2P content id) or null + * state 'verified' hash computed on save and equal to the server's + * 'unverified' hash computed, but the server sent none to compare + * 'unhashed' saved before hashing existed / main-thread fallback + * The P2P client (p2p-client.js) reads this to report holdings; OPFS stays the + * source of truth for what is playable (a record without a file is ignored). + * Every call resolves (null / [] on failure) — private windows can block IDB. + * See docs/p2p-architecture.md. + * ========================================================================== */ +(function () { + 'use strict'; + + const DB_NAME = 'ytp-device'; + const VERSION = 1; + let _open = null; + + function open() { + if (!_open) { + _open = new Promise((resolve, reject) => { + const r = indexedDB.open(DB_NAME, VERSION); + r.onupgradeneeded = () => { + const d = r.result; + if (!d.objectStoreNames.contains('files')) { + const s = d.createObjectStore('files', { keyPath: 'videoId' }); + s.createIndex('cid', 'cid', { unique: false }); + } + }; + r.onsuccess = () => resolve(r.result); + r.onerror = () => reject(r.error); + r.onblocked = () => reject(new Error('device-db blocked')); + }); + _open.catch(() => { _open = null; }); + } + return _open; + } + + const done = (req) => new Promise((resolve, reject) => { + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); + + async function store(mode) { + const d = await open(); + return d.transaction('files', mode).objectStore('files'); + } + + async function safe(fn, fallback) { + try { return await fn(); } catch { return fallback; } + } + + window.DeviceDB = { + isSupported: () => typeof indexedDB !== 'undefined', + putFile: (rec) => safe(async () => { + if (!rec || !rec.videoId) return null; + await done((await store('readwrite')).put({ + videoId: String(rec.videoId), + cid: rec.cid || null, + size: Number(rec.size) || 0, + savedAt: Number(rec.savedAt) || Date.now(), + lastCheckedAt: Number(rec.lastCheckedAt) || Date.now(), + state: rec.state || 'unhashed', + })); + return true; + }, null), + getFile: (videoId) => safe(async () => (await done((await store('readonly')).get(String(videoId)))) || null, null), + getByCid: (cid) => safe(async () => (await done((await store('readonly')).index('cid').get(String(cid)))) || null, null), + listFiles: () => safe(async () => (await done((await store('readonly')).getAll())) || [], []), + deleteFile: (videoId) => safe(async () => { await done((await store('readwrite')).delete(String(videoId))); return true; }, null), + clear: () => safe(async () => { await done((await store('readwrite')).clear()); return true; }, null), + }; +}()); diff --git a/frontend/index.html b/frontend/index.html index 0c5b624..f0ddef3 100755 --- a/frontend/index.html +++ b/frontend/index.html @@ -562,6 +562,7 @@ + diff --git a/frontend/opfs-worker.js b/frontend/opfs-worker.js index 6ec24b0..36b716b 100644 --- a/frontend/opfs-worker.js +++ b/frontend/opfs-worker.js @@ -13,12 +13,20 @@ * Out messages: * { type: 'unsupported' } → caller falls back to main thread * { type: 'progress', received } → bytes written so far - * { type: 'done', ext } → file stored as . + * { type: 'done', ext, sha256, expectedSha, size } + * → file stored as .; + * sha256 = hash of the stored bytes + * (P2P content id), expectedSha = the + * server's X-Content-SHA256 or null * { type: 'error', error } → failed; .part cleaned up * ========================================================================== */ 'use strict'; +// Incremental SHA-256 (frontend/sha256.js): the file is hashed while it is +// written, so the device knows its content id without re-reading the file. +try { importScripts('/sha256.js'); } catch { /* hashing unavailable — save still works */ } + async function getVideosDir() { const root = await navigator.storage.getDirectory(); return root.getDirectoryHandle('videos', { create: true }); @@ -60,6 +68,7 @@ self.onmessage = async (e) => { dir = await getVideosDir(); const partHandle = await dir.getFileHandle(partName, { create: true }); const access = await partHandle.createSyncAccessHandle(); + const hasher = self.Sha256 ? self.Sha256.create() : null; let offset = 0; try { const reader = res.body.getReader(); @@ -67,6 +76,7 @@ self.onmessage = async (e) => { const { done, value } = await reader.read(); if (done) break; access.write(value, { at: offset }); + if (hasher) hasher.update(value); offset += value.byteLength; self.postMessage({ type: 'progress', received: offset }); } @@ -81,6 +91,14 @@ self.onmessage = async (e) => { if (expected > 0 && offset !== expected) { throw new Error(`download cut short (${offset} of ${expected} bytes)`); } + // The server names the hash of what it sent (media cache copies). A + // mismatch means the bytes were damaged on the way — never keep them. + const sha256 = hasher ? hasher.hex() : null; + const sent = (res.headers.get('x-content-sha256') || '').trim().toLowerCase(); + const expectedSha = /^[0-9a-f]{64}$/.test(sent) ? sent : null; + if (sha256 && expectedSha && sha256 !== expectedSha) { + throw new Error('integrity check failed (content hash mismatch)'); + } // Finalize: .part → permanent name. Prefer the native rename, but treat // ANY move() failure as "unavailable" and fall back to a chunked copy — @@ -111,7 +129,7 @@ self.onmessage = async (e) => { await dir.removeEntry(partName); } - self.postMessage({ type: 'done', ext }); + self.postMessage({ type: 'done', ext, sha256, expectedSha, size: offset }); } catch (err) { // Never leave a corrupt partial behind try { if (dir && partName) await dir.removeEntry(partName); } catch { /* gone */ } diff --git a/frontend/opfs.js b/frontend/opfs.js index 341cc9c..e1a18a4 100644 --- a/frontend/opfs.js +++ b/frontend/opfs.js @@ -136,7 +136,7 @@ }; worker.onmessage = (e) => { const m = e.data || {}; - if (m.type === 'done') finish({ ok: true }); + if (m.type === 'done') finish({ ok: true, sha256: m.sha256 || null, expectedSha: m.expectedSha || null, size: m.size || 0 }); else if (m.type === 'unsupported') finish({ ok: false, fallback: true }); else if (m.type === 'error') finish({ ok: false, error: m.error }); // 'progress' messages are informational; ignored here diff --git a/frontend/sw.js b/frontend/sw.js index 13981d2..f51c99b 100644 --- a/frontend/sw.js +++ b/frontend/sw.js @@ -65,6 +65,7 @@ const SHELL = [ '/lyrics-core.js', '/stats-core.js', '/sha256.js', + '/device-db.js', '/app.js', '/manifest.webmanifest', '/icons/icon-192.png', diff --git a/plans/INDEX.md b/plans/INDEX.md index cf2265b..c9419dc 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -19,7 +19,7 @@ green, app boots with no JS errors, P2P on by default, offline boot works). | 009 | 009-server-content-hash-186e7f | Hash every validated server copy and register it as verified content | done | Hash every validated server copy and register it as verified content | uses plans/patches/009-* | | 010 | 010-views-and-retention-d0c6ca | Count views and evict server copies by retention criteria before LRU | done | Count views and evict server copies by retention criteria before LRU | | | 011 | 011-browser-sha256-e1793d | Add an incremental SHA-256 library for the browser and node tests | done | Add an incremental SHA-256 library for the browser and node tests | | -| 012 | 012-device-file-registry-288d55 | Add the on-device IndexedDB file registry and hash saves while downloading | in-progress | | browser harness | +| 012 | 012-device-file-registry-288d55 | Add the on-device IndexedDB file registry and hash saves while downloading | done | Add the on-device IndexedDB file registry and hash saves while downloading | browser harness | | 013 | 013-device-identity-and-holdings-3ba493 | Register devices and report verified holdings to the server | queued | | persistent holders, no TTL | | 014 | 014-p2p-presence-hub-ceced8 | Add the /ws/p2p presence and signalling hub and the holders endpoint | queued | | stale flag, never hidden | | 015 | 015-availability-ui-and-settings-3b9397 | Show peer availability with stale markers and add Sharing settings | queued | | | diff --git a/plans/active/012-device-file-registry-288d55.md b/plans/done/012-device-file-registry-288d55.md similarity index 89% rename from plans/active/012-device-file-registry-288d55.md rename to plans/done/012-device-file-registry-288d55.md index 3f889fc..0f86773 100644 --- a/plans/active/012-device-file-registry-288d55.md +++ b/plans/done/012-device-file-registry-288d55.md @@ -213,3 +213,10 @@ Do not commit. Do not push. Do not touch files outside the Steps. }; }()); ``` + +## Execution log + +- Executor: in-session Agent (haiku). Attempts: 1. Fix rounds: 0. +- Orchestrator re-ran Verification: `opfs-worker.js`, `opfs.js` and `device-db.js` byte-identical to the pre-tested versions; all frontend files pass `node --check`; 56 frontend tests pass; browser check in Chromium reproduced the expected JSON (good hash saved and `verified`, wrong hash rejected with `integrity check failed`, file without a hash saved, records created/read/deleted). +- Orchestrator note: the executor's first harness server was left running and answered my first check; killed it and reran on a clean port. Its pasted worker diff showed escaped backticks that do not exist in the real file. +- Executor Findings (verbatim): All plan steps executed successfully. Patch applied cleanly. Syntax valid, 56 unit tests pass (0 fail). device-db.js created and referenced in both index.html and sw.js. Browser check confirms integrity validation working: hash verification succeeds for correct files, integrity mismatch rejected ("bad"), unhashed files saved with "unhashed" state, device-db records created/retrieved/deleted properly.