josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resumable saves: ranged downloads that survive dropped connections, reloads and app kills

Browse Source 
Server
- /api/download/:id answers Range with a strong ETag ("<id>.<gen>") and honours
  If-Range; a stale partial gets the whole current file (streamed, so Bun does
  not re-apply the Range itself). Uploads get the same treatment.
- GET /api/download/:id/prepare never blocks: ready {gen,size,sha256,etag,ext},
  working (server still fetching), legacy (HEVC / too long / cache offline),
  failed. Recently refused prepares are remembered for 10 minutes.

Browser
- The OPFS worker saves in 8 MiB ranges, writes at the byte offset, flushes
  each chunk, retries each chunk 6 times with backoff (30 s idle timeout) and
  keeps the .part plus a .part.json sidecar naming the server copy it belongs
  to. A changed copy restarts cleanly; the finished file is hashed once and
  checked against the server's SHA-256.
- SaveQueue remembers unfinished saves and resumes them on start, online,
  return to the foreground and every 2 minutes while visible; one at a time.
- Downloads shows live MB progress, 'Preparing on server', 'Verifying', and
  paused saves with Resume and Cancel (confirmed).
- listVideos ignores the sidecars; new listPartials/discardPartial helpers.

Verified in Chromium through a connection-dropping proxy: paused at 8 MiB,
auto-resumed after a reload from byte 8388608, final SHA-256 matched.
This commit is contained in:
Jonathan Sykes
2026-10-03 00:46:52 +08:00
parent c8601c5953
commit 56b3f4e449
10 changed files with 698 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/media-cache.js
View File

@@ -648,6 +648,8 @@ export function createMediaCache({
const row = await db.getMedia(id);
return {
status: job ? job.status : row ? row.status : 'none',
gen: row ? row.gen : null,
sha256: row && row.sha256 ? row.sha256 : null,
size: row ? row.size : 0,
height: row ? row.height : 0,
vcodec: row ? row.vcodec : null,

98
server/server.js
View File

@@ -42,7 +42,7 @@ import { createHash } from 'node:crypto';
import { brotliCompressSync, constants as zlibConstants } from 'node:zlib';
import { initDb, upsertUser, recordVideoAccess, getUserData, createProfile, getProfile, saveProfile, createSharedPlaylist, getSharedPlaylist, queueInboxPlaylist, listInbox, deleteInboxItem, countInbox,
getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats } from './db.js';
import { createMediaCache, HIGH, LOW, validateMedia } from './media-cache.js';
import { createMediaCache, HIGH, LOW, validateMedia, MediaSkip } from './media-cache.js';
import * as notesDb from './db.js';
import { registerNoteRoutes, parseLrc, sanitizeLyrics } from './notes.js';
import { createRemoteHub } from './remote.js';
@@ -1262,16 +1262,27 @@ function cachedStreamsPayload(videoId, row) {
}
// Serve a file with byte-Range support (the <video> element seeks with it).
function rangeFileResponse(c, path, contentType, cacheControl) {
// `etag` (strong, quoted) lets a resuming client send If-Range: when the file
// changed under it, the Range is ignored and the whole new file comes back
// with 200, so stale and fresh bytes are never stitched together.
function rangeFileResponse(c, path, contentType, cacheControl, { etag = null, headers: extra = {} } = {}) {
const file = Bun.file(path);
const total = file.size;
const headers = {
'Content-Type': contentType,
'Accept-Ranges': 'bytes',
'Cache-Control': cacheControl,
...(etag ? { ETag: etag } : {}),
...extra,
};
const range = c.req.header('range');
if (!range) return new Response(file, { status: 200, headers });
let range = c.req.header('range');
const ifRange = c.req.header('if-range');
if (range && ifRange && etag && ifRange.trim() !== etag) {
// Stale partial: send the whole current file. As a stream, because Bun
// applies the request's Range to a Bun.file body on its own.
return new Response(file.stream(), { status: 200, headers: { ...headers, 'Content-Length': String(total) } });
}
if (!range) return new Response(file, { status: 200, headers: { ...headers, 'Content-Length': String(total) } });
const m = /^bytes=(\d*)-(\d*)$/.exec(range.trim());
let start, end;
if (m && m[1] !== '') {
@@ -1524,23 +1535,73 @@ app.post('/api/media/:id/redownload', async (c) => {
});
// Stream the server-cached copy as a save (OPFS writes it on the device).
function cachedDownloadResponse(videoId, fp, row) {
const file = Bun.file(`${MEDIA_DIR}/${videoId}.${row.gen}.mp4`);
if (fp) recordVideoAccess(fp, { id: videoId }).catch(() => {});
// Saves fetch the copy in byte ranges (docs/resumable-downloads-plan.md):
// the ETag pins the generation, so a resumed save never mixes two copies.
const prepareSkips = new Map(); // videoId → { reason, at } for /prepare
const DOWNLOAD_EXPOSE = 'X-Content-SHA256, Content-Range, Content-Length, ETag, Accept-Ranges';
function cachedDownloadResponse(c, videoId, fp, row) {
if (fp && !c.req.header('range')) recordVideoAccess(fp, { id: videoId }).catch(() => {});
media.touch(videoId);
return new Response(file, {
status: 200,
return rangeFileResponse(c, `${MEDIA_DIR}/${videoId}.${row.gen}.mp4`, 'video/mp4', 'no-store', {
etag: `"${videoId}.${row.gen}"`,
headers: {
'Content-Type': 'video/mp4',
'Content-Length': String(file.size),
'Content-Disposition': `attachment; filename="${videoId}.mp4"`,
'Cache-Control': 'no-store',
'Access-Control-Allow-Origin': '*',
...(row.sha256 ? { 'X-Content-SHA256': row.sha256, 'Access-Control-Expose-Headers': 'X-Content-SHA256' } : {}),
'Access-Control-Expose-Headers': DOWNLOAD_EXPOSE,
...(row.sha256 ? { 'X-Content-SHA256': row.sha256 } : {}),
},
});
}
// GET /api/download/:videoId/prepare[?hevc=1] — never blocks. Starts (or
// joins) the server-side fetch and reports where it is:
// ready → { gen, size, sha256 }: fetch /api/download/:id in ranges
// working → poll again (the server is still getting it from YouTube)
// legacy → this copy can't be served in ranges (too long for the cache,
// HEVC the device can't play, cache offline): use the old
// single-request save
app.get('/api/download/:videoId/prepare', async (c) => {
const videoId = (c.req.param('videoId') || '').trim();
const nocache = { 'Cache-Control': 'no-store' };
if (isUpload(videoId)) {
const u = await notesDb.getUpload(videoId);
if (!u) return c.json({ ok: false, error: 'upload not found' }, 404);
let size = 0;
try { size = Bun.file(uploads.filePath(u)).size; } catch { /* missing */ }
return c.json({ ok: true, state: 'ready', gen: 0, size, sha256: null, etag: `"${u.id}"`, ext: u.ext }, 200, nocache);
}
if (!media.isMediaId(videoId)) return c.json({ ok: false, error: 'invalid video id' }, 400);
const hevcOk = c.req.query('hevc') === '1';
const ready = await media.getReady(videoId);
if (ready) {
if (!servableTo(ready, hevcOk)) return c.json({ ok: true, state: 'legacy', reason: 'hevc' }, 200, nocache);
media.touch(videoId);
return c.json({ ok: true, state: 'ready', gen: ready.gen, size: ready.size, sha256: ready.sha256 || null,
etag: `"${videoId}.${ready.gen}"`, ext: 'mp4' }, 200, nocache);
}
// A job refused moments ago (too long for the cache, cache offline…) would
// just be refused again — answer from memory instead of re-probing YouTube.
const prior = prepareSkips.get(videoId);
if (prior && Date.now() - prior.at < 10 * 60_000) return c.json({ ok: true, state: 'legacy', reason: prior.reason }, 200, nocache);
// Start or join the job; its result is picked up by the next poll.
let skip = null;
media.ensureCached(videoId, { priority: HIGH }).catch((err) => {
skip = err;
if (err instanceof MediaSkip) {
prepareSkips.set(videoId, { reason: err.message, at: Date.now() });
if (prepareSkips.size > 2000) prepareSkips.clear();
}
});
await new Promise((r) => setTimeout(r, 0));
if (skip) {
const legacy = skip instanceof MediaSkip;
return c.json({ ok: legacy, state: legacy ? 'legacy' : 'failed', reason: skip.message }, legacy ? 200 : 500, nocache);
}
const st = await media.status(videoId);
if (st.status === 'failed') return c.json({ ok: true, state: 'legacy', reason: st.error || 'server fetch failed' }, 200, nocache);
return c.json({ ok: true, state: 'working', status: st.status }, 200, nocache);
});
// GET /api/download/:videoId
// Saves go through the server cache: the fetch runs as a server-owned job
// (a disconnecting client no longer kills it — the copy lands for next
@@ -1555,11 +1616,10 @@ app.get('/api/download/:videoId', async (c) => {
if (isUpload(videoId)) {
const u = await notesDb.getUpload(videoId);
if (!u) return c.json({ ok: false, error: 'upload not found' }, 404);
const f = Bun.file(uploads.filePath(u));
return new Response(f, { status: 200, headers: {
'Content-Type': u.mime, 'Content-Length': String(f.size),
'Content-Disposition': `attachment; filename="${u.id}.${u.ext}"`, 'Cache-Control': 'no-store',
} });
return rangeFileResponse(c, uploads.filePath(u), u.mime, 'no-store', {
etag: `"${u.id}"`,
headers: { 'Content-Disposition': `attachment; filename="${u.id}.${u.ext}"`, 'Access-Control-Expose-Headers': DOWNLOAD_EXPOSE },
});
}
const fp = c.req.query('fp');
@@ -1586,7 +1646,7 @@ app.get('/api/download/:videoId', async (c) => {
const row = await media.ensureCached(videoId, { priority: HIGH });
// A device that can't decode HEVC must not save the HEVC copy — give it
// the legacy H.264 save instead (the ?mux=1 path included).
if (servableTo(row, c.req.query('hevc') === '1')) return cachedDownloadResponse(videoId, fp, row);
if (servableTo(row, c.req.query('hevc') === '1')) return cachedDownloadResponse(c, videoId, fp, row);
cacheErr = Object.assign(new Error('cached copy is HEVC; client did not ask for it'), { code: 'SKIPPED' });
} catch (err) {
cacheErr = err;