From eef9fbf05fe2d29e100aa73e038c4ee90bd48398 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Sat, 17 Oct 2026 07:00:22 +0800 Subject: [PATCH] Still-picture songs save and play as audio plus one picture --- frontend/app-seams.test.js | 2 +- frontend/app.js | 61 ++++++++++++++++++++----- frontend/autosave.test.js | 2 +- frontend/opfs-worker.js | 2 +- frontend/opfs.js | 52 ++++++++++++++++++++++ frontend/p2p-client.js | 2 + server/db.js | 7 ++- server/media-cache.js | 91 ++++++++++++++++++++++++++++++++++---- server/media-cache.test.js | 21 ++++++++- server/server.js | 46 ++++++++++++++++++- 10 files changed, 262 insertions(+), 24 deletions(-) diff --git a/frontend/app-seams.test.js b/frontend/app-seams.test.js index a73effc..b98af93 100644 --- a/frontend/app-seams.test.js +++ b/frontend/app-seams.test.js @@ -5,7 +5,7 @@ const {createHash}=require('node:crypto'); const {join}=require('node:path'); const app=readFileSync(join(__dirname,'app.js'),'utf8'); test('Phase 4 preserves the exact playback, continuity, tracking and queue bodies',()=>{ - {const start=app.indexOf("const Player = {");assert.ok(start>=0,"Player");const body=app.slice(start,start+30343);assert.equal(createHash('sha256').update(body).digest('hex'),"190834ae6f33a168917416e55799f8a467e0d28f966f636a182745a59ffcac41","Player body unchanged");} + {const start=app.indexOf("const Player = {");assert.ok(start>=0,"Player");const body=app.slice(start,start+30964);assert.equal(createHash('sha256').update(body).digest('hex'),"5a63a7bc9ddb2266f9b88ab40730750aa092cd06106ef6a54e9560cd5d4605a0","Player body unchanged");} {const start=app.indexOf("const Transition = (() => {");assert.ok(start>=0,"Transition");const body=app.slice(start,start+8588);assert.equal(createHash('sha256').update(body).digest('hex'),"71997a537078308a41f0cbf8e29fd1658fb3a199c4f5a25d8af0931f1c2a6d50","Transition body unchanged");} {const start=app.indexOf("const Wave = (() => {");assert.ok(start>=0,"Wave");const body=app.slice(start,start+5241);assert.equal(createHash('sha256').update(body).digest('hex'),"7252f4e856c59aef8aabfcd394ce4bcba656110cbd01a1a8af662eeb48166a3c","Wave body unchanged");} {const start=app.indexOf("const StatsTrack = (() => {");assert.ok(start>=0,"StatsTrack");const body=app.slice(start,start+1462);assert.equal(createHash('sha256').update(body).digest('hex'),"9aae2631ba3bfad918319579b7120814af06c84949394d04e52c65345332d514","StatsTrack body unchanged");} diff --git a/frontend/app.js b/frontend/app.js index 40ffe12..8b02e88 100755 --- a/frontend/app.js +++ b/frontend/app.js @@ -412,10 +412,10 @@ const SaveQueue = (() => { // single-request save (HEVC copy this device can't play, too long for the // server cache, no worker/OPFS sync access), otherwise the save result — // { ok:false, paused:true } means the partial is kept and will resume. -async function resumableOpfsSave(videoId, qs, hevc, onProgress, signal) { +async function resumableOpfsSave(videoId, qs, prepQs, onProgress, signal) { if (typeof window.OPFS.downloadVideo !== 'function' || typeof Worker === 'undefined') return null; const enc = encodeURIComponent(videoId); - const prepUrl = `/api/download/${enc}/prepare${hevc ? '?hevc=1' : ''}`; + const prepUrl = `/api/download/${enc}/prepare${prepQs ? '?' + prepQs : ''}`; const dlUrl = `/api/download/${enc}${qs ? '?' + qs : ''}`; const report = (info) => { if (onProgress) { try { onProgress(info); } catch { /* UI only */ } } }; const started = Date.now(); @@ -463,6 +463,8 @@ async function opfsDownload(videoId, { mux = false, retry = false, onProgress = // stream exactly as before. if (mux) params.set('mux', '1'); if (hevcCapable()) params.set('hevc', '1'); + // Album-art songs come back as audio + picture instead of a video of one frame. + params.set('still', '1'); const qs = params.toString(); const url = `/api/download/${encodeURIComponent(videoId)}${qs ? '?' + qs : ''}`; @@ -475,7 +477,7 @@ async function opfsDownload(videoId, { mux = false, retry = false, onProgress = return { ok: false, paused: true, error: error.name === 'AbortError' ? 'Save paused' : 'Connection lost' }; } } - const resumed = await resumableOpfsSave(videoId, qs, params.get('hevc') === '1', onProgress, signal); + const resumed = await resumableOpfsSave(videoId, qs, params.get('hevc') === '1' ? 'hevc=1&still=1' : 'still=1', onProgress, signal); if (resumed) return resumed; // Preferred path: a dedicated Web Worker does the fetch AND the OPFS writes, @@ -499,7 +501,7 @@ async function opfsDownload(videoId, { mux = false, retry = false, onProgress = } // Determine file extension from Content-Type const ct = res.headers.get('content-type') || 'video/mp4'; - const ext = ct.includes('webm') ? 'webm' : ct.includes('ogg') ? 'ogg' : 'mp4'; + const ext = ct.includes('webm') ? 'webm' : ct.includes('ogg') ? 'ogg' : ct.includes('audio/mp4') ? 'm4a' : 'mp4'; await window.OPFS.writeFromResponse(videoId, ext, res); return { ok: true, cached: true }; } catch (err) { @@ -581,6 +583,7 @@ async function opfsClear() { // Blob URL for the currently playing OPFS video — revoked on next video load. let _currentBlobUrl = null; +let _currentArtUrl = null; // saved picture of a still-picture song (blob:) const API = { search: (query, { refresh = false, after = 0 } = {}) => WEB @@ -1953,6 +1956,30 @@ function recordDeviceFile(id, res) { .catch(() => {}); } +// Still-picture songs are saved as their audio (.m4a) plus one picture. +// True when this saved file is one; fetches its picture alongside. +async function saveStillArt(id) { + if (!WEB || !window.OPFS || isUploadId(id) || typeof window.OPFS.fileName !== 'function') return false; + const name = await window.OPFS.fileName(id).catch(() => null); + if (!name || !/\.m4a$/i.test(name)) return false; + try { + const r = await fetch(`/api/media/${encodeURIComponent(id)}/art`); + if (r.ok) await window.OPFS.saveArt(id, await r.arrayBuffer()); + } catch { /* the YouTube thumbnail stands in */ } + return true; +} + +// Playback fields for a saved still-picture song: play it as audio and show +// its saved picture. {} for anything else. +async function localStill(id) { + if (!WEB || !window.OPFS || isUploadId(id) || typeof window.OPFS.fileName !== 'function') return {}; + const name = await window.OPFS.fileName(id).catch(() => null); + if (!name || !/\.m4a$/i.test(name)) return {}; + const art = await window.OPFS.getArtUrl(id).catch(() => null); + if (art) { if (_currentArtUrl) window.OPFS.revokeUrl(_currentArtUrl); _currentArtUrl = art; } + return { kind: 'audio', still: true, artUrl: art || '' }; +} + // Download a video into the permanent offline cache. Safe to call repeatedly. async function preload(video, { quiet = false, mux = false, retry = false, directFallback = false } = {}) { const id = video.id; @@ -1998,7 +2025,9 @@ async function preload(video, { quiet = false, mux = false, retry = false, direc if (typeof EqRender !== 'undefined') EqRender.schedule(id); cachedIds.add(id); cacheMutations++; - recordDeviceFile(id, res); + // A still-picture song saved as audio isn't the server's video file, so + // it is never offered to other devices as one. + saveStillArt(id).then((still) => { if (!still) recordDeviceFile(id, res); }); warmThumb(thumbUrlFor(id, video)); if (!quiet) toast(`Saved “${video.title}” ✓`); } else if (!quiet && !(res && res.paused)) { @@ -2463,6 +2492,7 @@ const Player = { else URL.revokeObjectURL(_currentBlobUrl); _currentBlobUrl = null; } + if (_currentArtUrl) { if (window.OPFS) window.OPFS.revokeUrl(_currentArtUrl); _currentArtUrl = null; } try { // Party guests follow the host in real time: never hold playback for a // full device-to-device copy (and its accept prompt) — stream instead. @@ -2486,7 +2516,9 @@ const Player = { if (localUrl) { const eqAudioUrl = await useEqRender(videoObj.id); if (seq !== this._loadSeq) return; - current = { meta: { ...videoObj }, qualities: [], audioUrl: null, localUrl, eqAudioUrl }; + const still = await localStill(videoObj.id); + if (seq !== this._loadSeq) return; + current = { meta: { ...videoObj }, qualities: [], audioUrl: null, localUrl, eqAudioUrl, ...still }; this.afterLoad(); this.fallbackQueue = []; this.fbIndex = 0; this.attach(null); @@ -2517,7 +2549,9 @@ const Player = { if (localUrl) { _currentBlobUrl = localUrl; const eqAudioUrl = await useEqRender(videoObj.id); - current = { meta: { ...videoObj }, qualities: [], audioUrl: null, localUrl, eqAudioUrl }; + const still = await localStill(videoObj.id); + if (seq !== this._loadSeq) return; + current = { meta: { ...videoObj }, qualities: [], audioUrl: null, localUrl, eqAudioUrl, ...still }; this.afterLoad(); this.fallbackQueue = []; this.fbIndex = 0; this.attach(null); @@ -2533,6 +2567,9 @@ const Player = { if (current.serverCached && current.meta?.id) knownServerCachedIds.add(current.meta.id); // Merge richer metadata we may already have from the list card. current.meta = { ...videoObj, ...current.meta }; + // Album-art song: its video is one unchanging picture, so play the audio + // and show the picture — a fraction of the data for the same screen. + if (current.still && current.art) { current.kind = 'audio'; current.artUrl = current.art; } this.afterLoad(); const q = chooseQuality(); @@ -2757,7 +2794,7 @@ const Player = { A.src = current.eqAudioUrl || current.localUrl; V.removeAttribute('src'); V.load(); els.art.classList.remove('hidden'); - els.artImg.src = current.meta.thumbnail || ''; + els.artImg.src = current.artUrl || current.meta.thumbnail || ''; } else { this.mode = 'progressive'; this.master = V; @@ -2788,7 +2825,7 @@ const Player = { A.src = current.audioUrl || (quality && quality.url) || ''; V.removeAttribute('src'); V.load(); els.art.classList.remove('hidden'); - els.artImg.src = current.meta.thumbnail || ''; + els.artImg.src = current.artUrl || current.meta.thumbnail || ''; } else if (quality && quality.hasAudio) { // Progressive single stream (video already carries audio). this.mode = 'progressive'; @@ -7059,7 +7096,10 @@ async function exportToDevice(video) { say('Looking for the video…'); busy(); let file = await localFile(); - if (file) return ready(file); + // A still-picture song saved as audio: a desktop takes the server's video + // file when there is one; otherwise the audio is what gets saved. + const stillAudio = !!file && !isUploadId(id) && /\.m4a$/i.test(file.name || ''); + if (file && !stillAudio) return ready(file); // Is it on the server? (a 1-byte ranged probe; nothing is downloaded) let serverUrl = ''; @@ -7076,6 +7116,7 @@ async function exportToDevice(video) { button('Download file', true, () => { DE.download(serverUrl, DE.fileName(title, 'mp4', id)); done('Download started — see your Downloads folder'); }); return; } + if (stillAudio) return ready(file); // Other devices, only when the server can't help. if (!serverUrl && WEB && window.P2PTransfer && window.P2PCore && data.settings.p2pReceive !== false && !video.custom) { diff --git a/frontend/autosave.test.js b/frontend/autosave.test.js index 812c1f6..198bbde 100644 --- a/frontend/autosave.test.js +++ b/frontend/autosave.test.js @@ -16,7 +16,7 @@ function harness({ autoPreload = true, cached = [], result = { ok: true, cached: SaveSlots: { acquire: async () => {}, release() {} }, SaveQueue: { add: v => jobs.set(v.id, { status: 'queued' }), remove: id => jobs.delete(id), progress() {}, state: (id, status, message) => jobs.set(id, { status, message }) }, API: { cacheDownload: async id => { calls.push(id); if (error) throw error; return result; } }, - recordDeviceFile() {}, warmThumb() {}, thumbUrlFor: () => '', cacheMutations: 0, + recordDeviceFile() {}, saveStillArt: async () => false, warmThumb() {}, thumbUrlFor: () => '', cacheMutations: 0, }); const preload = source.slice(source.indexOf('async function preload(video,'), source.indexOf('// A device copy that failed to play.')); const playlists = source.slice(source.indexOf('function preloadPlaylist(pl)'), source.indexOf('// Update a single card\'s saved badge')); diff --git a/frontend/opfs-worker.js b/frontend/opfs-worker.js index a140730..11325f8 100644 --- a/frontend/opfs-worker.js +++ b/frontend/opfs-worker.js @@ -43,7 +43,7 @@ async function getVideosDir() { function extFromContentType(ct) { ct = ct || 'video/mp4'; - return ct.includes('webm') ? 'webm' : ct.includes('ogg') ? 'ogg' : 'mp4'; + return ct.includes('webm') ? 'webm' : ct.includes('ogg') ? 'ogg' : ct.includes('audio/mp4') ? 'm4a' : 'mp4'; } // .part → permanent name. Prefer the native rename, but treat ANY move() diff --git a/frontend/opfs.js b/frontend/opfs.js index b81b52c..248295b 100644 --- a/frontend/opfs.js +++ b/frontend/opfs.js @@ -52,6 +52,12 @@ const _blobUrls = new Set(); + // Pictures of still-picture songs saved as audio: art/.jpg + async function getArtDir() { + const root = await navigator.storage.getDirectory(); + return root.getDirectoryHandle('art', { create: true }); + } + window.OPFS = { // Does a file exist for this videoId? async hasVideo(videoId) { @@ -84,6 +90,50 @@ } }, + // The saved file's name (.), or null — an .m4a for a YouTube id + // is a still-picture song saved as audio + picture. + async fileName(videoId) { + try { + const found = await findHandle(videoId); + return found ? found[1] : null; + } catch { + return null; + } + }, + + // Store a still-picture song's picture. Main-thread writable first, else + // the worker's sync handle (Safari has no createWritable). + async saveArt(videoId, buffer) { + const dir = await getArtDir(); + const handle = await dir.getFileHandle(videoId + '.jpg', { create: true }); + if ('createWritable' in handle) { + const w = await handle.createWritable(); + await w.write(buffer); + await w.close(); + return; + } + await new Promise((resolve, reject) => { + const w = window.Lazy ? window.Lazy.worker('/opfs-worker.js') : new Worker('/opfs-worker.js'); + w.onmessage = (e) => { w.terminate(); (e.data && e.data.type === 'written') ? resolve() : reject(new Error((e.data && e.data.error) || 'write failed')); }; + w.onerror = () => { w.terminate(); reject(new Error('worker failed')); }; + w.postMessage({ op: 'write', dir: 'art', name: videoId + '.jpg', buffer }, [buffer]); + }); + }, + + // blob: URL of a saved picture, or null. Revoke with revokeUrl(). + async getArtUrl(videoId) { + try { + const dir = await getArtDir(); + const file = await (await dir.getFileHandle(videoId + '.jpg')).getFile(); + if (!file.size) return null; + const url = URL.createObjectURL(new Blob([file], { type: 'image/jpeg' })); + _blobUrls.add(url); + return url; + } catch { + return null; + } + }, + // Revoke an object URL previously returned by getFileUrl(). // 'hevc' | 'avc' | null for a stored mp4. Server copies are faststart, so // the sample description (hvc1/hev1 vs avc1) sits in the first megabyte. @@ -318,6 +368,7 @@ const dir = await getRoot(); await dir.removeEntry(name); } catch { /* already gone */ } + try { await (await getArtDir()).removeEntry(videoId + '.jpg'); } catch { /* no picture */ } }, // Delete all cached videos @@ -327,6 +378,7 @@ const names = []; for await (const [name] of dir.entries()) names.push(name); await Promise.all(names.map((n) => dir.removeEntry(n).catch(() => {}))); + try { await (await navigator.storage.getDirectory()).removeEntry('art', { recursive: true }); } catch { /* no pictures */ } // Revoke any outstanding blob URLs for (const url of _blobUrls) URL.revokeObjectURL(url); _blobUrls.clear(); diff --git a/frontend/p2p-client.js b/frontend/p2p-client.js index 5436575..da55533 100644 --- a/frontend/p2p-client.js +++ b/frontend/p2p-client.js @@ -101,6 +101,8 @@ } for (const f of byId.values()) { if (String(f.id).startsWith('edit_')) continue; // edited cuts are never shared + // A still-picture song saved as audio (.m4a) is not the server's file. + if (/\.m4a$/i.test(f.name) && !String(f.id).startsWith('upl_')) continue; const r = { videoId: f.id, cid: null, size: f.size, savedAt: Date.now(), lastCheckedAt: 0, state: 'unhashed' }; await window.DeviceDB.putFile(r); out.push({ ...r, name: f.name }); diff --git a/server/db.js b/server/db.js index 31271b1..1555b6d 100644 --- a/server/db.js +++ b/server/db.js @@ -291,6 +291,11 @@ export async function initDb() { await migrateVideoMeta(); await initLyricsSearch(); try { await db.execute('ALTER TABLE media_cache ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0'); } catch { /* already migrated */ } + // Still-picture songs: still = NULL (not checked yet) | 0 | 1. A still copy + // also has ..jpg, and devices may save just its m4a (audio_sha256). + for (const col of ['still INTEGER', 'audio_sha256 TEXT']) { + try { await db.execute(`ALTER TABLE media_cache ADD COLUMN ${col}`); } catch { /* already migrated */ } + } // Uploads shared from devices (PWA share target): who sent it, and whether it // is listed in everyone's search (admin uploads) or reachable only by link. for (const col of ['owner TEXT', 'listed INTEGER NOT NULL DEFAULT 1']) { @@ -675,7 +680,7 @@ const MEDIA_COLS = new Set([ 'status', 'gen', 'size', 'height', 'vcodec', 'acodec', 'duration', 'optimized', 'meta', 'priority', 'auto', 'attempts', 'error', 'retry_at', 'created_at', 'updated_at', 'last_access', 'hits', 'sha256', - 'pinned', + 'pinned', 'still', 'audio_sha256', ]); function rowToObj(res, row) { diff --git a/server/media-cache.js b/server/media-cache.js index 4199594..4652934 100644 --- a/server/media-cache.js +++ b/server/media-cache.js @@ -200,6 +200,9 @@ export function createMediaCache({ const optQueue = []; const optPending = new Set(); let optActive = null; // videoId currently being re-encoded + const stillQueue = []; // copies whose still-picture check hasn't run yet + const stillChecks = new Map(); // videoId -> running check (dedupes the two lanes) + let stillActive = false; const lastTouch = new Map(); function sweepTmp(prefix) { @@ -216,7 +219,7 @@ export function createMediaCache({ let names = []; try { names = readdirSync(dir); } catch { return; } for (const n of names) { - const m = /^(.+)\.(\d+)\.(mp4|m4a)$/.exec(n); + const m = /^(.+)\.(\d+)\.(mp4|m4a|jpg)$/.exec(n); if (!m || m[1] !== id || (keepGen !== null && Number(m[2]) === keepGen)) continue; const p = join(dir, n); const rm = () => { try { unlinkSync(p); } catch { /* gone */ } }; @@ -360,13 +363,14 @@ export function createMediaCache({ status: 'ready', gen, size: probe.size + sizeOf(fileFor(id, gen, 'm4a')), height: probe.height, vcodec: probe.vcodec, acodec: probe.acodec, duration: probe.duration, - optimized: 0, meta: JSON.stringify(metaFromInfo(info, probe.duration)), + optimized: 0, still: null, audio_sha256: null, meta: JSON.stringify(metaFromInfo(info, probe.duration)), attempts: 0, error: null, retry_at: 0, updated_at: t, last_access: t, sha256, }; await db.upsertMedia(id, fields); removeFiles(id, gen); job.status = 'ready'; log.info?.(`[media] cached ${id} ${probe.height}p ${fmtMB(fields.size)}`); + queueStill(id); enqueueOptimize(id); notifyReady(id, gen, sha256, probe, fields.meta); return { ...(prev || {}), video_id: id, ...fields }; @@ -428,7 +432,7 @@ export function createMediaCache({ }; // Very long videos would tie the lane up for hours on 2 niced threads. if (transcode.maxSeconds && row.duration > transcode.maxSeconds) { await markDone(); return; } - const still = row.duration >= STILL_MIN_DURATION && await isStill(src).catch(() => false); + const still = (await checkStill(id)) === 1; if (still) return optimizeStill(id, row, src, markDone); // Rev-2 copies already went through the HEVC lane; only the still check was new. if ((row.optimized || 0) >= 2) { await markDone(); return; } @@ -494,6 +498,8 @@ export function createMediaCache({ const oldAudio = fileFor(id, row.gen, 'm4a'); const newAudio = fileFor(id, gen, 'm4a'); try { linkSync(oldAudio, newAudio); } catch { copyFileSync(oldAudio, newAudio); } + const oldArt = fileFor(id, row.gen, 'jpg'); + if (existsSync(oldArt)) { try { linkSync(oldArt, fileFor(id, gen, 'jpg')); } catch { copyFileSync(oldArt, fileFor(id, gen, 'jpg')); } } await db.upsertMedia(id, { gen, size: probe.size + sizeOf(newAudio), height: probe.height, vcodec: probe.vcodec, duration: probe.duration, optimized: OPT_REV, updated_at: now(), sha256, @@ -503,12 +509,74 @@ export function createMediaCache({ log.info?.(`[media] optimized ${id} (${label}): ${fmtMB(oldSize)} → ${fmtMB(probe.size)} (${probe.vcodec})`); } + // Still-picture check, its own lane: keyframes only (~1 s a song), so it runs + // right after a copy lands instead of waiting behind the re-encode queue. + // A still copy gets its picture (..jpg) and the m4a's hash, which + // is what lets a device save audio + picture instead of the video. + function queueStill(id) { + if (!stillQueue.includes(id)) stillQueue.push(id); + pumpStill(); + } + + function pumpStill() { + if (stillActive || !stillQueue.length) return; + stillActive = true; + const id = stillQueue.shift(); + checkStill(id) + .catch((err) => log.warn?.(`[media] still check ${id} failed: ${err.message}`)) + .finally(() => { stillActive = false; pumpStill(); }); + } + + // Resolves to the row's still flag (1 | 0), or null when there is no copy. + function checkStill(id) { + if (stillChecks.has(id)) return stillChecks.get(id); + const p = (async () => { + const row = await db.getMedia(id); + if (!row || row.status !== 'ready') return null; + if (row.still === 0 || row.still === 1) return row.still; + const src = fileFor(id, row.gen); + if (!existsSync(src)) return null; + const fields = { still: 0 }; + const prefix = `${id}-${now()}-art`; + try { + if (row.duration >= STILL_MIN_DURATION && await isStill(src).catch(() => false)) { + const tmp = join(tmpDir, `${prefix}.jpg`); + const at = String(Math.min(30, Math.floor(row.duration / 2))); + const args = ['-v', 'error', '-nostdin', '-y', '-ss', at, '-i', src, '-frames:v', '1', + '-vf', 'scale=-2:min(720\\,ih)', '-q:v', '2', tmp]; + const f = nice ? await run(nice, ['-n', '19', ffmpeg, ...args]) : await run(ffmpeg, args); + if (f.code !== 0 || sizeOf(tmp) < 1024) throw new Error('picture grab failed: ' + tail(f.stderr)); + const audioSha = await hashFile(fileFor(id, row.gen, 'm4a')); + const cur = await db.getMedia(id); + if (!cur || cur.status !== 'ready' || cur.gen !== row.gen) return null; // replaced meanwhile + renameSync(tmp, fileFor(id, row.gen, 'jpg')); + Object.assign(fields, { still: 1, audio_sha256: audioSha }); + } + } catch (err) { + log.warn?.(`[media] still check ${id}: ${err.message}`); + return null; // left unchecked: retried at the next boot + } finally { + sweepTmp(prefix); + } + const cur = await db.getMedia(id); + if (!cur || cur.status !== 'ready' || cur.gen !== row.gen) return null; + await db.upsertMedia(id, fields); + if (fields.still) log.info?.(`[media] still picture ${id}: devices can save audio + picture`); + return fields.still; + })().finally(() => stillChecks.delete(id)); + stillChecks.set(id, p); + return p; + } + // Keyframes only: ~1 s for a 5-minute video. Every keyframe-to-keyframe // scene score must stay under STILL_MAX_SCENE. async function isStill(src) { const r = await run(ffmpeg, ['-hide_banner', '-nostdin', '-skip_frame', 'nokey', '-i', src, '-an', '-vf', "scale=160:-2,select='gte(scene\\,0)',metadata=print:key=lavfi.scene_score", '-f', 'null', '-']); if (r.code !== 0) return false; + // Our own still encode (one picture at 1 fps, a keyframe every 10 min) has + // too few keyframes to score; no YouTube upload is 1 fps. + if (/Stream #0:\d+[^\n]*Video:[^\n]*\b1 fps\b/.test(String(r.stderr))) return true; const scores = [...String(r.stderr).matchAll(/scene_score=([\d.]+)/g)].map((m) => Number(m[1])); return scores.length + 1 >= STILL_MIN_KEYFRAMES && scores.every((x) => x <= STILL_MAX_SCENE); } @@ -527,10 +595,16 @@ export function createMediaCache({ // a few seconds early, so read the audio stream's own length. const pa = await run(ffprobe, ['-v', 'error', '-select_streams', 'a:0', '-show_entries', 'stream=duration', '-of', 'csv=p=0', src]); const seconds = Number(String(pa.stdout).trim()) > 0 ? Number(String(pa.stdout).trim()) : row.duration; - const f = await niced(['-v', 'error', '-nostdin', '-y', '-ss', at, '-i', src, '-frames:v', '1', - '-vf', 'scale=-2:min(720\\,ih)', frame]); - if (f.code !== 0) throw new Error('still frame grab failed: ' + tail(f.stderr)); - const r = await niced(['-v', 'error', '-nostdin', '-y', '-loop', '1', '-framerate', '1', '-i', frame, '-i', src, + // The picture devices show (..jpg) is the frame to loop, so the + // saved-as-audio copy and the video copy look the same. + let picture = fileFor(id, row.gen, 'jpg'); + if (!existsSync(picture)) { + picture = frame; + const f = await niced(['-v', 'error', '-nostdin', '-y', '-ss', at, '-i', src, '-frames:v', '1', + '-vf', 'scale=-2:min(720\\,ih)', frame]); + if (f.code !== 0) throw new Error('still frame grab failed: ' + tail(f.stderr)); + } + const r = await niced(['-v', 'error', '-nostdin', '-y', '-loop', '1', '-framerate', '1', '-i', picture, '-i', src, '-map', '0:v', '-map', '1:a:0', '-t', String(seconds), '-c:v', 'libx264', '-preset', 'medium', '-crf', String(transcode.crf), '-tune', 'stillimage', '-profile:v', 'high', '-level:v', '4.0', '-pix_fmt', 'yuv420p', '-g', '600', '-threads', threads, @@ -747,7 +821,7 @@ export function createMediaCache({ const byId = new Map(rows.map((r) => [r.video_id, r])); for (const name of readdirSync(dir)) { if (name === '.tmp') continue; - const m = /^(.+)\.(\d+)\.(mp4|m4a)$/.exec(name); + const m = /^(.+)\.(\d+)\.(mp4|m4a|jpg)$/.exec(name); const r = m && byId.get(m[1]); if (!m || !r || r.status !== 'ready' || r.gen !== Number(m[2])) { try { unlinkSync(join(dir, name)); } catch { /* gone */ } @@ -757,6 +831,7 @@ export function createMediaCache({ for (const r of rows) { if (r.status === 'ready') { if (!readyFilesExist(r)) { await db.deleteMedia(r.video_id); removeFiles(r.video_id); continue; } + if (r.still === null || r.still === undefined) queueStill(r.video_id); if ((r.optimized || 0) < OPT_REV) enqueueOptimize(r.video_id); } else if (r.status === 'queued' || r.status === 'downloading' || r.status === 'validating') { startJob(r.video_id, r.priority === HIGH ? HIGH : LOW, !!r.auto, r); diff --git a/server/media-cache.test.js b/server/media-cache.test.js index e484be0..cbd2c42 100644 --- a/server/media-cache.test.js +++ b/server/media-cache.test.js @@ -404,7 +404,7 @@ describe('media cache jobs', () => { test('still-picture lane: one looped frame in H.264, far smaller, still verifies', async () => { await clearDb(); const lines = []; - const { cache } = makeCache({ fixture: 'still40.mp4', duration: 40, log: { info: (m) => lines.push(m), warn: (m) => lines.push(m) }, + const { cache, dir } = makeCache({ fixture: 'still40.mp4', duration: 40, log: { info: (m) => lines.push(m), warn: (m) => lines.push(m) }, transcode: { enabled: true, codec: 'hevc', crf: 28, preset: 'ultrafast', threads: 2 } }); await cache.init(); const first = await cache.ensureCached('stilAAAAAA1', { priority: HIGH }); @@ -415,6 +415,24 @@ describe('media cache jobs', () => { expect(after.size).toBeLessThan(first.size * 0.6); expect(lines.some((l) => /\(still\)/.test(l))).toBe(true); expect(await cache.verify('stilAAAAAA1')).toBe(true); + // Devices can save audio + picture: the flag, the picture and the m4a hash. + expect(after.still).toBe(1); + const art = await cache.filePath('stilAAAAAA1', after.gen, 'jpg'); + expect(art).toBeTruthy(); + expect(after.audio_sha256).toBe(await sha256File(await cache.filePath('stilAAAAAA1', after.gen, 'm4a'))); + // An already-looped copy (1 fps, one keyframe) is recognised as still after + // a deploy that predates the flag. + await dbmod.upsertMedia('stilAAAAAA1', { still: null, audio_sha256: null }); + unlinkSync(art); + const again = createMediaCache({ + dir, db: { getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia, + listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia, mediaStats: dbmod.mediaStats }, + log: silentLog, getInfo: async () => ({ duration: 40 }), download: async () => { throw new Error('no download expected'); }, + transcode: { enabled: false }, freeBytes: () => 100 * 1024 ** 3, + }); + await again.init(); + expect(await until(async () => (await dbmod.getMedia('stilAAAAAA1')).still === 1, 20000)).toBe(true); + expect(await again.filePath('stilAAAAAA1', after.gen, 'jpg')).toBeTruthy(); }, 90000); test('a moving copy already done by the HEVC lane is not re-encoded for rev 3', async () => { @@ -437,6 +455,7 @@ describe('media cache jobs', () => { await again.init(); expect(await until(async () => (await dbmod.getMedia('rev2AAAAAA1')).optimized === OPT_REV, 30000)).toBe(true); expect((await dbmod.getMedia('rev2AAAAAA1')).gen).toBe(before.gen); + expect((await dbmod.getMedia('rev2AAAAAA1')).still).toBe(0); }, 120000); test('copies evaluated by an older lane revision are re-queued at boot', async () => { diff --git a/server/server.js b/server/server.js index 56374d8..ad71124 100644 --- a/server/server.js +++ b/server/server.js @@ -1404,9 +1404,20 @@ function cachedStreamsPayload(videoId, row) { }], serverCached: true, cid: row.sha256 || null, + // Album-art song: the picture never changes, so the app plays the audio + // and shows this instead of streaming video (additive, web-only). + ...(row.still === 1 && existsSync(`${MEDIA_DIR}/${videoId}.${row.gen}.jpg`) + ? { still: true, art: `/api/media/${videoId}/art?g=${row.gen}` } : {}), }; } +// A still-picture copy whose m4a a device may save instead of the video. +function stillAudioPath(videoId, row) { + if (!row || row.still !== 1 || !row.audio_sha256) return null; + const p = `${MEDIA_DIR}/${videoId}.${row.gen}.m4a`; + return existsSync(p) && existsSync(`${MEDIA_DIR}/${videoId}.${row.gen}.jpg`) ? p : null; +} + // Serve a file with byte-Range support (the