Complete verified offline assets in a resumable worker job

frontend/asset-sync-core.js

@@ -6,7 +6,7 @@
   const url = (path, file) => path + '?v=' + file.h;
 
   function paths(manifest) {
-    return [...new Set(Object.values(manifest.groups).flatMap(group => group.files))].sort();
+    return Object.keys(manifest.files).sort();
   }
 
   function blocking(manifest, previous, activeLayout) {
@@ -55,13 +55,33 @@
       return !!response && response.headers.get('X-Asset-Hash') === manifest.files[path].h;
     }));
     const missing = found.filter(value => !value).length;
-    return { ready: missing === 0, missing, version: manifest.buildTag };
+    const offline = await completeness(manifest, cache);
+    return { ready: missing === 0, missing, version: manifest.buildTag, ...offline };
   }
 
-  async function syncAssets(manifest, { cache, fetchFn, concurrency = 6, attempts = 3, activeLayout, groups }) {
+  // All concurrent messages/layout jobs in this worker share six download slots.
+  let downloads = 0;
+  const waiters = [];
+  async function downloadSlot(task) {
+    if (downloads >= 6) await new Promise(resolve => waiters.push(resolve));
+    else downloads++;
+    try { return await task(); }
+    finally { const next = waiters.shift(); if (next) next(); else downloads--; }
+  }
+
+  const pendingDownloads = new Map();
+  async function sharedDownload(key, task) {
+    while (pendingDownloads.has(key)) await pendingDownloads.get(key).catch(() => {});
+    const promise = downloadSlot(task);
+    pendingDownloads.set(key, promise);
+    try { return await promise; }
+    finally { pendingDownloads.delete(key); }
+  }
+
+  async function syncAssets(manifest, { cache, fetchFn, concurrency = 6, attempts = 3, activeLayout, groups, all = false }) {
     const previous = (await state(cache))?.current;
     const missing = [];
-    const selected = groups ? [...new Set(groups.flatMap(name => manifest.groups[name]?.files || []))].sort()
+    const selected = all ? paths(manifest) : groups ? [...new Set(groups.flatMap(name => manifest.groups[name]?.files || []))].sort()
       : blocking(manifest, previous, activeLayout);
     for (const path of selected) {
       const key = url(path, manifest.files[path]);
@@ -71,8 +91,11 @@
     const count = missing.length;
     let failure;
     await Promise.all(Array.from({ length: Math.max(1, Math.min(6, concurrency, count)) }, async () => {
-      while (missing.length && !failure) {
+      while (missing.length) {
         const key = missing.shift();
+        try { await sharedDownload(key, async () => {
+        const held = await cache.match(key);
+        if (held?.headers.get('X-Asset-Hash') === key.split('v=')[1]) return;
         let error;
         for (let attempt = 0; attempt < Math.min(3, attempts); attempt++) {
           const controller = new AbortController();
@@ -89,14 +112,59 @@
           } catch (err) { error = err; }
           finally { clearTimeout(timer); }
         }
-        if (error) failure = error;
+        if (error) throw error;
+        }); } catch (error) { failure = error; }
       }
     }));
     if (failure) throw failure;
-    if (!groups && !(await status(manifest, cache, { previous, activeLayout })).ready) throw new Error('Incomplete blocking assets');
+    if (!groups && !all && !(await status(manifest, cache, { previous, activeLayout })).ready) throw new Error('Incomplete blocking assets');
     return { refreshed: count, caches: 1 };
   }
 
+
+  // Verified cache entries are the durable progress journal. Counters are rebuilt
+  // after worker termination or browser eviction; never trust a stored ready bit.
+  async function completeness(manifest, cache) {
+    const missingFiles = [];
+    for (const path of paths(manifest)) {
+      const response = await cache.match(url(path, manifest.files[path]));
+      if (!response || response.headers.get('X-Asset-Hash') !== manifest.files[path].h) missingFiles.push(path);
+    }
+    const total = paths(manifest).length;
+    return { offlineReady: missingFiles.length === 0, cached: total - missingFiles.length, total, missingFiles };
+  }
+
+  const completionKey = manifest => '/__ytp_completion/' + manifest.buildTag;
+  async function complete(manifest, { cache, fetchFn, cycles = 4, now = Date.now,
+    sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), baseDelay = 1000,
+    maxDelay = 30000, notify = () => {}, force = false } = {}) {
+    const key = completionKey(manifest), saved = await cache.match(key);
+    let job = saved ? await saved.json() : { failures: 0, nextRetryAt: 0 };
+    let snapshot = await completeness(manifest, cache);
+    const publish = async running => {
+      job = { ...job, ...snapshot, running, version: manifest.buildTag };
+      await cache.put(key, Response.json(job));
+      await notify(job);
+    };
+    if (snapshot.offlineReady) { job.failures = 0; job.nextRetryAt = 0; await publish(false); return job; }
+    if (!force && job.nextRetryAt > now()) await sleep(Math.min(maxDelay, job.nextRetryAt - now()));
+    for (let cycle = 0; cycle < cycles; cycle++) {
+      await publish(true);
+      let error;
+      try { await syncAssets(manifest, { cache, fetchFn, all: true }); }
+      catch (failure) { error = failure; }
+      snapshot = await completeness(manifest, cache);
+      if (snapshot.offlineReady) { job.failures = 0; job.nextRetryAt = 0; job.error = null; await publish(false); return job; }
+      job.failures++;
+      const delay = Math.min(maxDelay, baseDelay * 2 ** Math.min(job.failures - 1, 10));
+      job.nextRetryAt = now() + delay;
+      job.error = String(error?.message || 'Incomplete offline cache');
+      await publish(cycle + 1 < cycles);
+      if (cycle + 1 < cycles) await sleep(delay);
+    }
+    return job;
+  }
+
   async function commit(manifest, cache, { previousClients, activeLayout } = {}) {
     const old = await state(cache);
     if (!(await status(manifest, cache, { previous: old?.current, activeLayout })).ready) {
@@ -119,7 +187,7 @@
     return next;
   }
 
-  const api = { CACHE, STATE, url, paths, blocking, plan, retained, fallback, status, syncAssets, state, commit };
+  const api = { CACHE, STATE, url, paths, blocking, plan, retained, fallback, status, completeness, completionKey, complete, syncAssets, state, commit };
   if (typeof module !== 'undefined' && module.exports) module.exports = api;
   else root.AssetSyncCore = api;
 })(typeof globalThis !== 'undefined' ? globalThis : this);