Still-picture songs save and play as audio plus one picture

server/db.js

@@ -291,6 +291,11 @@ export async function initDb() {
   await migrateVideoMeta();
   await initLyricsSearch();
   try { await db.execute('ALTER TABLE media_cache ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0'); } catch { /* already migrated */ }
+  // Still-picture songs: still = NULL (not checked yet) | 0 | 1. A still copy
+  // also has <id>.<gen>.jpg, and devices may save just its m4a (audio_sha256).
+  for (const col of ['still INTEGER', 'audio_sha256 TEXT']) {
+    try { await db.execute(`ALTER TABLE media_cache ADD COLUMN ${col}`); } catch { /* already migrated */ }
+  }
   // Uploads shared from devices (PWA share target): who sent it, and whether it
   // is listed in everyone's search (admin uploads) or reachable only by link.
   for (const col of ['owner TEXT', 'listed INTEGER NOT NULL DEFAULT 1']) {
@@ -675,7 +680,7 @@ const MEDIA_COLS = new Set([
   'status', 'gen', 'size', 'height', 'vcodec', 'acodec', 'duration', 'optimized',
   'meta', 'priority', 'auto', 'attempts', 'error', 'retry_at', 'created_at',
   'updated_at', 'last_access', 'hits', 'sha256',
-  'pinned',
+  'pinned', 'still', 'audio_sha256',
 ]);
 
 function rowToObj(res, row) {

server/media-cache.js

@@ -200,6 +200,9 @@ export function createMediaCache({
   const optQueue = [];
   const optPending = new Set();
   let optActive = null;       // videoId currently being re-encoded
+  const stillQueue = [];      // copies whose still-picture check hasn't run yet
+  const stillChecks = new Map(); // videoId -> running check (dedupes the two lanes)
+  let stillActive = false;
   const lastTouch = new Map();
 
   function sweepTmp(prefix) {
@@ -216,7 +219,7 @@ export function createMediaCache({
     let names = [];
     try { names = readdirSync(dir); } catch { return; }
     for (const n of names) {
-      const m = /^(.+)\.(\d+)\.(mp4|m4a)$/.exec(n);
+      const m = /^(.+)\.(\d+)\.(mp4|m4a|jpg)$/.exec(n);
       if (!m || m[1] !== id || (keepGen !== null && Number(m[2]) === keepGen)) continue;
       const p = join(dir, n);
       const rm = () => { try { unlinkSync(p); } catch { /* gone */ } };
@@ -360,13 +363,14 @@ export function createMediaCache({
         status: 'ready', gen,
         size: probe.size + sizeOf(fileFor(id, gen, 'm4a')),
         height: probe.height, vcodec: probe.vcodec, acodec: probe.acodec, duration: probe.duration,
-        optimized: 0, meta: JSON.stringify(metaFromInfo(info, probe.duration)),
+        optimized: 0, still: null, audio_sha256: null, meta: JSON.stringify(metaFromInfo(info, probe.duration)),
         attempts: 0, error: null, retry_at: 0, updated_at: t, last_access: t, sha256,
       };
       await db.upsertMedia(id, fields);
       removeFiles(id, gen);
       job.status = 'ready';
       log.info?.(`[media] cached ${id} ${probe.height}p ${fmtMB(fields.size)}`);
+      queueStill(id);
       enqueueOptimize(id);
       notifyReady(id, gen, sha256, probe, fields.meta);
       return { ...(prev || {}), video_id: id, ...fields };
@@ -428,7 +432,7 @@ export function createMediaCache({
     };
     // Very long videos would tie the lane up for hours on 2 niced threads.
     if (transcode.maxSeconds && row.duration > transcode.maxSeconds) { await markDone(); return; }
-    const still = row.duration >= STILL_MIN_DURATION && await isStill(src).catch(() => false);
+    const still = (await checkStill(id)) === 1;
     if (still) return optimizeStill(id, row, src, markDone);
     // Rev-2 copies already went through the HEVC lane; only the still check was new.
     if ((row.optimized || 0) >= 2) { await markDone(); return; }
@@ -494,6 +498,8 @@ export function createMediaCache({
     const oldAudio = fileFor(id, row.gen, 'm4a');
     const newAudio = fileFor(id, gen, 'm4a');
     try { linkSync(oldAudio, newAudio); } catch { copyFileSync(oldAudio, newAudio); }
+    const oldArt = fileFor(id, row.gen, 'jpg');
+    if (existsSync(oldArt)) { try { linkSync(oldArt, fileFor(id, gen, 'jpg')); } catch { copyFileSync(oldArt, fileFor(id, gen, 'jpg')); } }
     await db.upsertMedia(id, {
       gen, size: probe.size + sizeOf(newAudio), height: probe.height, vcodec: probe.vcodec,
       duration: probe.duration, optimized: OPT_REV, updated_at: now(), sha256,
@@ -503,12 +509,74 @@ export function createMediaCache({
     log.info?.(`[media] optimized ${id} (${label}): ${fmtMB(oldSize)} → ${fmtMB(probe.size)} (${probe.vcodec})`);
   }
 
+  // Still-picture check, its own lane: keyframes only (~1 s a song), so it runs
+  // right after a copy lands instead of waiting behind the re-encode queue.
+  // A still copy gets its picture (<id>.<gen>.jpg) and the m4a's hash, which
+  // is what lets a device save audio + picture instead of the video.
+  function queueStill(id) {
+    if (!stillQueue.includes(id)) stillQueue.push(id);
+    pumpStill();
+  }
+
+  function pumpStill() {
+    if (stillActive || !stillQueue.length) return;
+    stillActive = true;
+    const id = stillQueue.shift();
+    checkStill(id)
+      .catch((err) => log.warn?.(`[media] still check ${id} failed: ${err.message}`))
+      .finally(() => { stillActive = false; pumpStill(); });
+  }
+
+  // Resolves to the row's still flag (1 | 0), or null when there is no copy.
+  function checkStill(id) {
+    if (stillChecks.has(id)) return stillChecks.get(id);
+    const p = (async () => {
+      const row = await db.getMedia(id);
+      if (!row || row.status !== 'ready') return null;
+      if (row.still === 0 || row.still === 1) return row.still;
+      const src = fileFor(id, row.gen);
+      if (!existsSync(src)) return null;
+      const fields = { still: 0 };
+      const prefix = `${id}-${now()}-art`;
+      try {
+        if (row.duration >= STILL_MIN_DURATION && await isStill(src).catch(() => false)) {
+          const tmp = join(tmpDir, `${prefix}.jpg`);
+          const at = String(Math.min(30, Math.floor(row.duration / 2)));
+          const args = ['-v', 'error', '-nostdin', '-y', '-ss', at, '-i', src, '-frames:v', '1',
+            '-vf', 'scale=-2:min(720\\,ih)', '-q:v', '2', tmp];
+          const f = nice ? await run(nice, ['-n', '19', ffmpeg, ...args]) : await run(ffmpeg, args);
+          if (f.code !== 0 || sizeOf(tmp) < 1024) throw new Error('picture grab failed: ' + tail(f.stderr));
+          const audioSha = await hashFile(fileFor(id, row.gen, 'm4a'));
+          const cur = await db.getMedia(id);
+          if (!cur || cur.status !== 'ready' || cur.gen !== row.gen) return null; // replaced meanwhile
+          renameSync(tmp, fileFor(id, row.gen, 'jpg'));
+          Object.assign(fields, { still: 1, audio_sha256: audioSha });
+        }
+      } catch (err) {
+        log.warn?.(`[media] still check ${id}: ${err.message}`);
+        return null; // left unchecked: retried at the next boot
+      } finally {
+        sweepTmp(prefix);
+      }
+      const cur = await db.getMedia(id);
+      if (!cur || cur.status !== 'ready' || cur.gen !== row.gen) return null;
+      await db.upsertMedia(id, fields);
+      if (fields.still) log.info?.(`[media] still picture ${id}: devices can save audio + picture`);
+      return fields.still;
+    })().finally(() => stillChecks.delete(id));
+    stillChecks.set(id, p);
+    return p;
+  }
+
   // Keyframes only: ~1 s for a 5-minute video. Every keyframe-to-keyframe
   // scene score must stay under STILL_MAX_SCENE.
   async function isStill(src) {
     const r = await run(ffmpeg, ['-hide_banner', '-nostdin', '-skip_frame', 'nokey', '-i', src, '-an',
       '-vf', "scale=160:-2,select='gte(scene\\,0)',metadata=print:key=lavfi.scene_score", '-f', 'null', '-']);
     if (r.code !== 0) return false;
+    // Our own still encode (one picture at 1 fps, a keyframe every 10 min) has
+    // too few keyframes to score; no YouTube upload is 1 fps.
+    if (/Stream #0:\d+[^\n]*Video:[^\n]*\b1 fps\b/.test(String(r.stderr))) return true;
     const scores = [...String(r.stderr).matchAll(/scene_score=([\d.]+)/g)].map((m) => Number(m[1]));
     return scores.length + 1 >= STILL_MIN_KEYFRAMES && scores.every((x) => x <= STILL_MAX_SCENE);
   }
@@ -527,10 +595,16 @@ export function createMediaCache({
       // a few seconds early, so read the audio stream's own length.
       const pa = await run(ffprobe, ['-v', 'error', '-select_streams', 'a:0', '-show_entries', 'stream=duration', '-of', 'csv=p=0', src]);
       const seconds = Number(String(pa.stdout).trim()) > 0 ? Number(String(pa.stdout).trim()) : row.duration;
-      const f = await niced(['-v', 'error', '-nostdin', '-y', '-ss', at, '-i', src, '-frames:v', '1',
-        '-vf', 'scale=-2:min(720\\,ih)', frame]);
-      if (f.code !== 0) throw new Error('still frame grab failed: ' + tail(f.stderr));
-      const r = await niced(['-v', 'error', '-nostdin', '-y', '-loop', '1', '-framerate', '1', '-i', frame, '-i', src,
+      // The picture devices show (<id>.<gen>.jpg) is the frame to loop, so the
+      // saved-as-audio copy and the video copy look the same.
+      let picture = fileFor(id, row.gen, 'jpg');
+      if (!existsSync(picture)) {
+        picture = frame;
+        const f = await niced(['-v', 'error', '-nostdin', '-y', '-ss', at, '-i', src, '-frames:v', '1',
+          '-vf', 'scale=-2:min(720\\,ih)', frame]);
+        if (f.code !== 0) throw new Error('still frame grab failed: ' + tail(f.stderr));
+      }
+      const r = await niced(['-v', 'error', '-nostdin', '-y', '-loop', '1', '-framerate', '1', '-i', picture, '-i', src,
         '-map', '0:v', '-map', '1:a:0', '-t', String(seconds),
         '-c:v', 'libx264', '-preset', 'medium', '-crf', String(transcode.crf), '-tune', 'stillimage',
         '-profile:v', 'high', '-level:v', '4.0', '-pix_fmt', 'yuv420p', '-g', '600', '-threads', threads,
@@ -747,7 +821,7 @@ export function createMediaCache({
     const byId = new Map(rows.map((r) => [r.video_id, r]));
     for (const name of readdirSync(dir)) {
       if (name === '.tmp') continue;
-      const m = /^(.+)\.(\d+)\.(mp4|m4a)$/.exec(name);
+      const m = /^(.+)\.(\d+)\.(mp4|m4a|jpg)$/.exec(name);
       const r = m && byId.get(m[1]);
       if (!m || !r || r.status !== 'ready' || r.gen !== Number(m[2])) {
         try { unlinkSync(join(dir, name)); } catch { /* gone */ }
@@ -757,6 +831,7 @@ export function createMediaCache({
     for (const r of rows) {
       if (r.status === 'ready') {
         if (!readyFilesExist(r)) { await db.deleteMedia(r.video_id); removeFiles(r.video_id); continue; }
+        if (r.still === null || r.still === undefined) queueStill(r.video_id);
         if ((r.optimized || 0) < OPT_REV) enqueueOptimize(r.video_id);
       } else if (r.status === 'queued' || r.status === 'downloading' || r.status === 'validating') {
         startJob(r.video_id, r.priority === HIGH ? HIGH : LOW, !!r.auto, r);

server/media-cache.test.js

@@ -404,7 +404,7 @@ describe('media cache jobs', () => {
   test('still-picture lane: one looped frame in H.264, far smaller, still verifies', async () => {
     await clearDb();
     const lines = [];
-    const { cache } = makeCache({ fixture: 'still40.mp4', duration: 40, log: { info: (m) => lines.push(m), warn: (m) => lines.push(m) },
+    const { cache, dir } = makeCache({ fixture: 'still40.mp4', duration: 40, log: { info: (m) => lines.push(m), warn: (m) => lines.push(m) },
       transcode: { enabled: true, codec: 'hevc', crf: 28, preset: 'ultrafast', threads: 2 } });
     await cache.init();
     const first = await cache.ensureCached('stilAAAAAA1', { priority: HIGH });
@@ -415,6 +415,24 @@ describe('media cache jobs', () => {
     expect(after.size).toBeLessThan(first.size * 0.6);
     expect(lines.some((l) => /\(still\)/.test(l))).toBe(true);
     expect(await cache.verify('stilAAAAAA1')).toBe(true);
+    // Devices can save audio + picture: the flag, the picture and the m4a hash.
+    expect(after.still).toBe(1);
+    const art = await cache.filePath('stilAAAAAA1', after.gen, 'jpg');
+    expect(art).toBeTruthy();
+    expect(after.audio_sha256).toBe(await sha256File(await cache.filePath('stilAAAAAA1', after.gen, 'm4a')));
+    // An already-looped copy (1 fps, one keyframe) is recognised as still after
+    // a deploy that predates the flag.
+    await dbmod.upsertMedia('stilAAAAAA1', { still: null, audio_sha256: null });
+    unlinkSync(art);
+    const again = createMediaCache({
+      dir, db: { getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia,
+        listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia, mediaStats: dbmod.mediaStats },
+      log: silentLog, getInfo: async () => ({ duration: 40 }), download: async () => { throw new Error('no download expected'); },
+      transcode: { enabled: false }, freeBytes: () => 100 * 1024 ** 3,
+    });
+    await again.init();
+    expect(await until(async () => (await dbmod.getMedia('stilAAAAAA1')).still === 1, 20000)).toBe(true);
+    expect(await again.filePath('stilAAAAAA1', after.gen, 'jpg')).toBeTruthy();
   }, 90000);
 
   test('a moving copy already done by the HEVC lane is not re-encoded for rev 3', async () => {
@@ -437,6 +455,7 @@ describe('media cache jobs', () => {
     await again.init();
     expect(await until(async () => (await dbmod.getMedia('rev2AAAAAA1')).optimized === OPT_REV, 30000)).toBe(true);
     expect((await dbmod.getMedia('rev2AAAAAA1')).gen).toBe(before.gen);
+    expect((await dbmod.getMedia('rev2AAAAAA1')).still).toBe(0);
   }, 120000);
 
   test('copies evaluated by an older lane revision are re-queued at boot', async () => {

server/server.js

@@ -1404,9 +1404,20 @@ function cachedStreamsPayload(videoId, row) {
     }],
     serverCached: true,
     cid: row.sha256 || null,
+    // Album-art song: the picture never changes, so the app plays the audio
+    // and shows this instead of streaming video (additive, web-only).
+    ...(row.still === 1 && existsSync(`${MEDIA_DIR}/${videoId}.${row.gen}.jpg`)
+      ? { still: true, art: `/api/media/${videoId}/art?g=${row.gen}` } : {}),
   };
 }
 
+// A still-picture copy whose m4a a device may save instead of the video.
+function stillAudioPath(videoId, row) {
+  if (!row || row.still !== 1 || !row.audio_sha256) return null;
+  const p = `${MEDIA_DIR}/${videoId}.${row.gen}.m4a`;
+  return existsSync(p) && existsSync(`${MEDIA_DIR}/${videoId}.${row.gen}.jpg`) ? p : null;
+}
+
 // Serve a file with byte-Range support (the <video> element seeks with it).
 // `etag` (strong, quoted) lets a resuming client send If-Range: when the file
 // changed under it, the Range is ignored and the whole new file comes back
@@ -1470,6 +1481,17 @@ app.get('/api/media/:id', async (c) => {
     g ? 'public, max-age=31536000, immutable' : 'no-cache');
 });
 
+// GET /api/media/:id/art?g=<gen> — the picture of a still-picture song
+// (one frame, ≤720p JPEG), shown while its audio plays.
+app.get('/api/media/:id/art', async (c) => {
+  const id = c.req.param('id');
+  const g = c.req.query('g');
+  const path = await media.filePath(id, g, 'jpg');
+  if (!path) return c.json({ ok: false, error: 'no picture' }, 404);
+  return new Response(Bun.file(path), { headers: { 'Content-Type': 'image/jpeg',
+    'Cache-Control': g ? 'public, max-age=31536000, immutable' : 'no-cache' } });
+});
+
 // GET /api/export/:id?name=<file name>[&hevc=1] — the server's copy as a file
 // download (Content-Disposition: attachment), Range-aware so a big file can
 // resume. Serves a cached YouTube copy or one of the server's own uploads, so a
@@ -1719,6 +1741,18 @@ const DOWNLOAD_EXPOSE = 'X-Content-SHA256, Content-Range, Content-Length, ETag,
 function cachedDownloadResponse(c, videoId, fp, row) {
   if (fp && !c.req.header('range')) recordVideoAccess(fp, { id: videoId }).catch(() => {});
   media.touch(videoId);
+  const stillAudio = c.req.query('still') === '1' ? stillAudioPath(videoId, row) : null;
+  if (stillAudio) {
+    return rangeFileResponse(c, stillAudio, 'audio/mp4', 'no-store', {
+      etag: `"${videoId}.${row.gen}.a"`,
+      headers: {
+        'Content-Disposition': `attachment; filename="${videoId}.m4a"`,
+        'Access-Control-Allow-Origin': '*',
+        'Access-Control-Expose-Headers': DOWNLOAD_EXPOSE,
+        'X-Content-SHA256': row.audio_sha256,
+      },
+    });
+  }
   return rangeFileResponse(c, `${MEDIA_DIR}/${videoId}.${row.gen}.mp4`, 'video/mp4', 'no-store', {
     etag: `"${videoId}.${row.gen}"`,
     headers: {
@@ -1761,6 +1795,14 @@ app.get('/api/download/:videoId/prepare', async (c) => {
   const hevcOk = c.req.query('hevc') === '1';
   const ready = await media.getReady(videoId);
   if (ready) {
+    // ?still=1: the device would rather keep the audio + picture of an
+    // album-art song than a video of one unchanging frame.
+    const stillAudio = c.req.query('still') === '1' ? stillAudioPath(videoId, ready) : null;
+    if (stillAudio) {
+      media.touch(videoId);
+      return c.json({ ok: true, state: 'ready', gen: ready.gen, size: Bun.file(stillAudio).size, sha256: ready.audio_sha256,
+        etag: `"${videoId}.${ready.gen}.a"`, ext: 'm4a', still: true, art: `/api/media/${videoId}/art?g=${ready.gen}` }, 200, nocache);
+    }
     if (!servableTo(ready, hevcOk)) return c.json({ ok: true, state: 'legacy', reason: 'hevc' }, 200, nocache);
     media.touch(videoId);
     // The row's size can differ from the mp4 on disk (it is not the file the
@@ -1839,7 +1881,9 @@ app.get('/api/download/:videoId', async (c) => {
     const row = await media.ensureCached(videoId, { priority: HIGH });
     // A device that can't decode HEVC must not save the HEVC copy — give it
     // the legacy H.264 save instead (the ?mux=1 path included).
-    if (servableTo(row, c.req.query('hevc') === '1')) return cachedDownloadResponse(c, videoId, fp, row);
+    if (servableTo(row, c.req.query('hevc') === '1') || stillAudioPath(videoId, row) && c.req.query('still') === '1') {
+      return cachedDownloadResponse(c, videoId, fp, row);
+    }
     cacheErr = Object.assign(new Error('cached copy is HEVC; client did not ask for it'), { code: 'SKIPPED' });
   } catch (err) {
     cacheErr = err;