Resumable saves: ranged downloads that survive dropped connections, reloads and app kills
Server
- /api/download/:id answers Range with a strong ETag ("<id>.<gen>") and honours
If-Range; a stale partial gets the whole current file (streamed, so Bun does
not re-apply the Range itself). Uploads get the same treatment.
- GET /api/download/:id/prepare never blocks: ready {gen,size,sha256,etag,ext},
working (server still fetching), legacy (HEVC / too long / cache offline),
failed. Recently refused prepares are remembered for 10 minutes.
Browser
- The OPFS worker saves in 8 MiB ranges, writes at the byte offset, flushes
each chunk, retries each chunk 6 times with backoff (30 s idle timeout) and
keeps the .part plus a .part.json sidecar naming the server copy it belongs
to. A changed copy restarts cleanly; the finished file is hashed once and
checked against the server's SHA-256.
- SaveQueue remembers unfinished saves and resumes them on start, online,
return to the foreground and every 2 minutes while visible; one at a time.
- Downloads shows live MB progress, 'Preparing on server', 'Verifying', and
paused saves with Resume and Cancel (confirmed).
- listVideos ignores the sidecars; new listPartials/discardPartial helpers.
Verified in Chromium through a connection-dropping proxy: paused at 8 MiB,
auto-resumed after a reload from byte 8388608, final SHA-256 matched.
This commit is contained in:
Jonathan Sykes
201
frontend/app.js
201
frontend/app.js
@@ -270,7 +270,127 @@ const SearchLibrary = (() => {
|
||||
// OPFS bridge wrappers — return the same shape as the Tauri cache_* commands
|
||||
// so the rest of app.js works without changes.
|
||||
|
||||
async function opfsDownload(videoId, { mux = false } = {}) {
|
||||
// ---------- Save queue: resumable saves survive reloads and dropped links ----------
|
||||
// Every save is recorded here when it starts and removed when it finishes or
|
||||
// fails for good. A save that only paused (partial kept on the device) stays
|
||||
// listed and is resumed when the app opens, comes back online or returns to
|
||||
// the foreground — and every 2 minutes while the app is visible.
|
||||
const SaveQueue = (() => {
|
||||
const KEY = 'ytpSaveQueue';
|
||||
const MAX_AGE = 7 * 24 * 3600 * 1000;
|
||||
const prog = new Map();
|
||||
let running = false;
|
||||
const load = () => { try { return JSON.parse(localStorage.getItem(KEY) || '{}') || {}; } catch { return {}; } };
|
||||
const store = (q) => { try { localStorage.setItem(KEY, JSON.stringify(q)); } catch { /* storage blocked */ } };
|
||||
|
||||
function add(v) {
|
||||
if (!v || !v.id || v.custom) return;
|
||||
const q = load();
|
||||
q[v.id] = { id: v.id, title: v.title || '', channel: v.channel || '', thumbnail: v.thumbnail || '',
|
||||
duration: v.duration || 0, channelUrl: v.channelUrl || '', channelId: v.channelId || '',
|
||||
at: (q[v.id] && q[v.id].at) || Date.now() };
|
||||
store(q);
|
||||
}
|
||||
function remove(id) { const q = load(); if (q[id]) { delete q[id]; store(q); } }
|
||||
const pending = () => Object.values(load());
|
||||
const paused = () => pending().filter((v) => !downloading.has(v.id) && !cachedIds.has(v.id));
|
||||
const get = (id) => prog.get(id) || null;
|
||||
|
||||
function label(p) {
|
||||
if (!p) return '⏳ Saving for offline…';
|
||||
const mb = (n) => (n / 1048576).toFixed(n >= 1048576 * 100 ? 0 : 1);
|
||||
if (p.phase === 'preparing') return `⏳ Preparing on server… ${Math.round((p.elapsed || 0) / 1000)} s`;
|
||||
if (p.phase === 'verifying') return '✓ Verifying…';
|
||||
if (p.total) return `⬇ ${mb(p.received)} / ${mb(p.total)} MB`;
|
||||
return '⏳ Saving for offline…';
|
||||
}
|
||||
function paint(id) {
|
||||
const row = document.querySelector(`.card.downloading[data-id="${CSS.escape(id)}"]`);
|
||||
if (!row) return;
|
||||
const p = get(id);
|
||||
const ch = row.querySelector('.card-channel');
|
||||
if (ch) ch.textContent = label(p);
|
||||
const bar = row.querySelector('.dl-bar');
|
||||
if (bar) {
|
||||
const pct = p && p.total ? Math.max(1, Math.min(100, (p.received / p.total) * 100)) : null;
|
||||
bar.classList.toggle('determinate', pct !== null);
|
||||
bar.style.width = pct !== null ? pct + '%' : '';
|
||||
}
|
||||
}
|
||||
function progress(id, p) { if (p) prog.set(id, p); else prog.delete(id); paint(id); }
|
||||
|
||||
async function resumeAll() {
|
||||
if (running || !navigator.onLine) return;
|
||||
running = true;
|
||||
try {
|
||||
for (const v of pending()) {
|
||||
if (cachedIds.has(v.id)) { remove(v.id); continue; }
|
||||
if (downloading.has(v.id)) continue;
|
||||
if (Date.now() - v.at > MAX_AGE) { cancel(v.id); continue; }
|
||||
await preload(v, { quiet: true }); // one at a time: the homelab link is the bottleneck
|
||||
if (!navigator.onLine) break;
|
||||
}
|
||||
} finally { running = false; }
|
||||
if (view.type === 'downloads') renderList();
|
||||
}
|
||||
function cancel(id) {
|
||||
remove(id);
|
||||
prog.delete(id);
|
||||
if (window.OPFS && window.OPFS.discardPartial) window.OPFS.discardPartial(id);
|
||||
}
|
||||
function start() {
|
||||
setTimeout(resumeAll, 4000);
|
||||
window.addEventListener('online', () => resumeAll());
|
||||
document.addEventListener('visibilitychange', () => { if (document.visibilityState === 'visible') resumeAll(); });
|
||||
setInterval(() => { if (document.visibilityState === 'visible' && paused().length) resumeAll(); }, 120000);
|
||||
}
|
||||
return { add, remove, pending, paused, progress, get, label, resumeAll, cancel, start };
|
||||
})();
|
||||
|
||||
// Resumable save (docs/resumable-downloads-plan.md): ask the server to get
|
||||
// the video first (never blocks), then fetch its finished copy in byte ranges
|
||||
// inside the worker. Returns null when this video must use the legacy
|
||||
// single-request save (HEVC copy this device can't play, too long for the
|
||||
// server cache, no worker/OPFS sync access), otherwise the save result —
|
||||
// { ok:false, paused:true } means the partial is kept and will resume.
|
||||
async function resumableOpfsSave(videoId, qs, hevc, onProgress) {
|
||||
if (typeof window.OPFS.downloadVideo !== 'function' || typeof Worker === 'undefined') return null;
|
||||
const enc = encodeURIComponent(videoId);
|
||||
const prepUrl = `/api/download/${enc}/prepare${hevc ? '?hevc=1' : ''}`;
|
||||
const dlUrl = `/api/download/${enc}${qs ? '?' + qs : ''}`;
|
||||
const report = (info) => { if (onProgress) { try { onProgress(info); } catch { /* UI only */ } } };
|
||||
const started = Date.now();
|
||||
for (let round = 0; round < 2; round++) {
|
||||
let prep = null;
|
||||
for (;;) {
|
||||
let j;
|
||||
try {
|
||||
const r = await fetch(prepUrl, { cache: 'no-store' });
|
||||
j = await r.json();
|
||||
} catch (err) {
|
||||
return { ok: false, paused: true, error: 'connection lost' };
|
||||
}
|
||||
if (j.state === 'ready') { prep = j; break; }
|
||||
if (j.state === 'legacy') return null;
|
||||
if (!j.ok || j.state === 'failed') return { ok: false, error: j.reason || j.error || 'the server could not get this video' };
|
||||
report({ phase: 'preparing', elapsed: Date.now() - started });
|
||||
if (Date.now() - started > 3 * 3600 * 1000) return { ok: false, paused: true, error: 'the server is still preparing it' };
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
const w = await window.OPFS.downloadVideo(videoId, dlUrl, {
|
||||
resumable: { etag: prep.etag, size: prep.size, sha256: prep.sha256 || null, ext: prep.ext || 'mp4' },
|
||||
onProgress: (received, total, m) => report({ phase: m && m.verifying ? 'verifying' : 'downloading', received, total }),
|
||||
});
|
||||
if (w.ok) return { ok: true, cached: true, sha256: w.sha256 || null, expectedSha: w.expectedSha || null, size: w.size || 0 };
|
||||
if (w.changed) continue; // the server copy was replaced — start over on the new one
|
||||
if (w.paused) return { ok: false, paused: true, error: w.error || 'connection lost', received: w.received || 0 };
|
||||
if (w.fallback) return null;
|
||||
return { ok: false, error: w.error || 'save failed' };
|
||||
}
|
||||
return { ok: false, error: 'the server copy kept changing — try again' };
|
||||
}
|
||||
|
||||
async function opfsDownload(videoId, { mux = false, onProgress = null } = {}) {
|
||||
if (!window.OPFS || !window.OPFS.isSupported()) {
|
||||
return { ok: false, error: 'OPFS not supported in this browser' };
|
||||
}
|
||||
@@ -285,6 +405,9 @@ async function opfsDownload(videoId, { mux = false } = {}) {
|
||||
const qs = params.toString();
|
||||
const url = `/api/download/${encodeURIComponent(videoId)}${qs ? '?' + qs : ''}`;
|
||||
|
||||
const resumed = await resumableOpfsSave(videoId, qs, params.get('hevc') === '1', onProgress);
|
||||
if (resumed) return resumed;
|
||||
|
||||
// Preferred path: a dedicated Web Worker does the fetch AND the OPFS writes,
|
||||
// so a big save never touches the main thread (no UI jank, no audio
|
||||
// stutter). ANY worker failure — unsupported API or a mid-download error —
|
||||
@@ -1428,20 +1551,29 @@ async function preload(video, { quiet = false, mux = false } = {}) {
|
||||
// finishes in the `finally` below.
|
||||
if (current && current.meta && current.meta.id === id) updateNowPlayingActions();
|
||||
if (!quiet) toast(`Saving “${video.title}” for offline…`);
|
||||
SaveQueue.add(video);
|
||||
try {
|
||||
const res = await API.cacheDownload(id, { mux });
|
||||
const res = await API.cacheDownload(id, { mux, onProgress: (p) => SaveQueue.progress(id, p) });
|
||||
if (res && res.paused) {
|
||||
// The partial stays on the device; SaveQueue picks it up again.
|
||||
if (!quiet) toast(`Paused “${video.title}” — it will continue when the connection is back`);
|
||||
} else {
|
||||
SaveQueue.remove(id);
|
||||
}
|
||||
if (res && res.ok && res.cached) {
|
||||
cachedIds.add(id);
|
||||
cacheMutations++;
|
||||
recordDeviceFile(id, res);
|
||||
warmThumb(thumbUrlFor(id, video));
|
||||
if (!quiet) toast(`Saved “${video.title}” ✓`);
|
||||
} else if (!quiet) {
|
||||
} else if (!quiet && !(res && res.paused)) {
|
||||
toast('⚠ ' + ((res && res.error) || 'Could not save video'));
|
||||
}
|
||||
} catch (e) {
|
||||
SaveQueue.remove(id);
|
||||
if (!quiet) toast('⚠ Saving not supported in this build.');
|
||||
} finally {
|
||||
SaveQueue.progress(id, null);
|
||||
downloading.delete(id);
|
||||
downloadMeta.delete(id);
|
||||
markCardCacheState(id, cachedIds.has(id) ? 'cached' : 'none');
|
||||
@@ -8311,7 +8443,8 @@ function renderDownloads() {
|
||||
c.innerHTML = '';
|
||||
|
||||
const active = [...downloadMeta.values()];
|
||||
if (!active.length) {
|
||||
const paused = SaveQueue.paused();
|
||||
if (!active.length && !paused.length) {
|
||||
const empty = document.createElement('div');
|
||||
empty.className = 'empty-state';
|
||||
empty.innerHTML = `
|
||||
@@ -8329,25 +8462,73 @@ function renderDownloads() {
|
||||
|
||||
const note = document.createElement('div');
|
||||
note.className = 'saved-summary';
|
||||
note.innerHTML = `<span class="saved-total">${active.length}</span><span class="saved-sub">download${active.length === 1 ? '' : 's'} in progress</span>`;
|
||||
const n = active.length + paused.length;
|
||||
note.innerHTML = `<span class="saved-total">${n}</span><span class="saved-sub">download${n === 1 ? '' : 's'}${paused.length ? ` · ${paused.length} paused` : ' in progress'}</span>`;
|
||||
c.appendChild(note);
|
||||
|
||||
active.forEach((v) => {
|
||||
const rowFor = (v, isPaused) => {
|
||||
const row = document.createElement('div');
|
||||
row.className = 'card downloading';
|
||||
row.className = 'card downloading' + (isPaused ? ' dl-paused' : '');
|
||||
row.dataset.id = v.id;
|
||||
row.innerHTML = `
|
||||
<div class="thumb">
|
||||
<img loading="lazy" decoding="async" src="${escapeHtml(v.thumbnail || '')}" alt="" />
|
||||
<img loading="lazy" decoding="async" src="${escapeHtml(v.thumbnail || thumbUrlFor(v.id, v) || '')}" alt="" />
|
||||
<div class="dl-progress"><div class="dl-bar"></div></div>
|
||||
</div>
|
||||
<div class="card-info">
|
||||
<div class="card-title"></div>
|
||||
<div class="card-channel">⏳ Saving for offline…</div>
|
||||
<div class="card-channel"></div>
|
||||
</div>`;
|
||||
row.querySelector('.card-title').textContent = v.title || v.id;
|
||||
return row;
|
||||
};
|
||||
|
||||
active.forEach((v) => {
|
||||
c.appendChild(rowFor(v, false));
|
||||
// paint the live label/bar now (the worker keeps updating it)
|
||||
SaveQueue.progress(v.id, SaveQueue.get(v.id));
|
||||
});
|
||||
|
||||
paused.forEach((v) => {
|
||||
const row = rowFor(v, true);
|
||||
row.querySelector('.card-channel').textContent = '⏸ Paused — continues when the connection is back';
|
||||
const acts = document.createElement('div');
|
||||
acts.className = 'dl-actions';
|
||||
const resume = document.createElement('button');
|
||||
resume.type = 'button';
|
||||
resume.className = 'dl-act';
|
||||
resume.textContent = '▶ Resume';
|
||||
resume.onclick = (e) => { e.stopPropagation(); preload(v, { quiet: true }); renderList(); };
|
||||
const cancel = document.createElement('button');
|
||||
cancel.type = 'button';
|
||||
cancel.className = 'dl-act danger';
|
||||
cancel.textContent = 'Cancel';
|
||||
cancel.onclick = (e) => {
|
||||
e.stopPropagation();
|
||||
showModal(`Cancel saving “${v.title || v.id}”?`, document.createTextNode('The part already downloaded will be deleted.'), [
|
||||
{ label: 'Keep', onClick: closeModal },
|
||||
{ label: 'Cancel save', danger: true, onClick: () => { SaveQueue.cancel(v.id); closeModal(); renderList(); } },
|
||||
]);
|
||||
};
|
||||
acts.append(resume, cancel);
|
||||
row.querySelector('.card-info').appendChild(acts);
|
||||
c.appendChild(row);
|
||||
});
|
||||
|
||||
// Fill in how far each paused save got (read from the partial on disk).
|
||||
if (paused.length && window.OPFS && window.OPFS.listPartials) {
|
||||
window.OPFS.listPartials().then((parts) => {
|
||||
for (const p of parts) {
|
||||
const row = c.querySelector(`.card.dl-paused[data-id="${CSS.escape(p.id)}"]`);
|
||||
if (!row || !p.size) continue;
|
||||
const bar = row.querySelector('.dl-bar');
|
||||
bar.classList.add('determinate');
|
||||
bar.style.width = Math.max(1, Math.min(100, (p.received / p.size) * 100)) + '%';
|
||||
row.querySelector('.card-channel').textContent =
|
||||
`⏸ Paused at ${(p.received / 1048576).toFixed(0)} / ${(p.size / 1048576).toFixed(0)} MB — continues when the connection is back`;
|
||||
}
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -11125,6 +11306,8 @@ async function boot() {
|
||||
}
|
||||
data.playlists.forEach(preloadPlaylist);
|
||||
preloadPinnedPlaylists();
|
||||
// Continue saves that were interrupted (closed app, dropped connection).
|
||||
if (WEB) SaveQueue.start();
|
||||
// Not awaited — artwork backfill must never delay first paint.
|
||||
warmOfflineThumbs();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user