IndexedDB search library (1000x100, 15-day copies, Update search), server video metadata for instant local results
This commit is contained in:
Claude
206
frontend/app.js
206
frontend/app.js
@@ -142,8 +142,8 @@ const RecentSearches = (() => {
|
||||
},
|
||||
clear() { save([]); }, // saved results stay in SearchLibrary (cleared only from Settings)
|
||||
// Cache the just-fetched result set for `q` (called on every successful search).
|
||||
cacheResults(q, results) { SearchLibrary.put(q, results); },
|
||||
// Cached {query, results, ts} for `q`, or null.
|
||||
cacheResults(q, results, ts) { SearchLibrary.put(q, results, ts); },
|
||||
// Promise of the cached {query, results, ts} for `q`, or null.
|
||||
getResults(q) {
|
||||
return SearchLibrary.get(q);
|
||||
},
|
||||
@@ -162,62 +162,108 @@ const RecentSearches = (() => {
|
||||
// The thumbnails themselves live in the service worker's `ytplayer-thumbs`
|
||||
// cache (warmed below), so a saved search renders with its pictures offline.
|
||||
const SearchLibrary = (() => {
|
||||
const KEY = 'ytpSearchLibrary';
|
||||
const LEGACY = 'ytpRecentSearchResults';
|
||||
const MAX_QUERIES = 150;
|
||||
const PER_QUERY = 50;
|
||||
const INDEX_KEY = 'ytpSearchIndex'; // small, synchronous: who/when/how big
|
||||
const OLD_KEYS = ['ytpSearchLibrary', 'ytpRecentSearchResults']; // earlier localStorage stores
|
||||
const MAX_QUERIES = 1000;
|
||||
const PER_QUERY = 100;
|
||||
const FRESH_MS = 15 * 24 * 3600_000; // a saved copy is trusted for 15 days
|
||||
const norm = (q) => String(q || '').trim().toLowerCase();
|
||||
let mem = null;
|
||||
function read() {
|
||||
if (mem) return mem;
|
||||
let o = {};
|
||||
try { o = JSON.parse(localStorage.getItem(KEY) || '{}') || {}; } catch { o = {}; }
|
||||
// One-time import of the old per-recent-search cache.
|
||||
|
||||
// Metadata index { key: { query, count, ts, bytes, thumb } } lives in memory and
|
||||
// localStorage (≈250 KB at 1000 searches); the result lists themselves — up to
|
||||
// 100 cards each — live in IndexedDB and are read one search at a time.
|
||||
let index = {};
|
||||
try { index = JSON.parse(localStorage.getItem(INDEX_KEY) || '{}') || {}; } catch { index = {}; }
|
||||
let persistTimer = 0;
|
||||
const persistIndex = () => {
|
||||
clearTimeout(persistTimer);
|
||||
persistTimer = setTimeout(() => { try { localStorage.setItem(INDEX_KEY, JSON.stringify(index)); } catch { /* quota */ } }, 300);
|
||||
};
|
||||
|
||||
// IndexedDB, with a memory fallback when it is blocked (private windows).
|
||||
const fallback = new Map();
|
||||
let dbp = null;
|
||||
const open = () => dbp || (dbp = new Promise((resolve) => {
|
||||
try {
|
||||
const old = JSON.parse(localStorage.getItem(LEGACY) || 'null');
|
||||
if (old && typeof old === 'object') {
|
||||
for (const [k, v] of Object.entries(old)) if (v && Array.isArray(v.results) && !o[k]) o[k] = v;
|
||||
localStorage.removeItem(LEGACY);
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
mem = o;
|
||||
return mem;
|
||||
const req = indexedDB.open('ytpSearchLibrary', 1);
|
||||
req.onupgradeneeded = () => req.result.createObjectStore('r', { keyPath: 'key' });
|
||||
req.onsuccess = () => resolve(req.result);
|
||||
req.onerror = req.onblocked = () => resolve(null);
|
||||
} catch { resolve(null); }
|
||||
}));
|
||||
async function tx(mode, fn) {
|
||||
const d = await open();
|
||||
if (!d) return fn(null);
|
||||
return new Promise((resolve) => {
|
||||
let out;
|
||||
try {
|
||||
const t = d.transaction('r', mode);
|
||||
out = fn(t.objectStore('r'));
|
||||
t.oncomplete = () => resolve(out && out.result !== undefined ? out.result : undefined);
|
||||
t.onerror = t.onabort = () => resolve(undefined);
|
||||
} catch { resolve(undefined); }
|
||||
});
|
||||
}
|
||||
function write() {
|
||||
const o = read();
|
||||
for (let attempt = 0; attempt < 4; attempt++) {
|
||||
try { localStorage.setItem(KEY, JSON.stringify(o)); return true; } catch {
|
||||
// Quota: drop the oldest fifth and retry.
|
||||
const keys = Object.keys(o).sort((a, b) => (o[a].ts || 0) - (o[b].ts || 0));
|
||||
keys.slice(0, Math.max(1, Math.ceil(keys.length / 5))).forEach((k) => delete o[k]);
|
||||
const idbPut = (rec) => tx('readwrite', (st) => st ? st.put(rec) : void fallback.set(rec.key, rec));
|
||||
const idbGet = (key) => tx('readonly', (st) => st ? st.get(key) : { result: fallback.get(key) });
|
||||
const idbDel = (keys) => tx('readwrite', (st) => { if (st) keys.forEach((k) => st.delete(k)); else keys.forEach((k) => fallback.delete(k)); });
|
||||
const idbClear = () => tx('readwrite', (st) => st ? st.clear() : void fallback.clear());
|
||||
|
||||
// One-time import of the earlier localStorage stores.
|
||||
(async () => {
|
||||
for (const k of OLD_KEYS) {
|
||||
let o = null;
|
||||
try { o = JSON.parse(localStorage.getItem(k) || 'null'); } catch { o = null; }
|
||||
if (o && typeof o === 'object') {
|
||||
for (const [key, v] of Object.entries(o)) {
|
||||
if (!v || !Array.isArray(v.results) || !v.results.length || index[key]) continue;
|
||||
put(v.query || key, v.results, v.ts);
|
||||
}
|
||||
}
|
||||
try { localStorage.removeItem(k); } catch { /* blocked */ }
|
||||
}
|
||||
return false;
|
||||
})();
|
||||
|
||||
function put(q, results, ts) {
|
||||
q = String(q || '').trim();
|
||||
if (!q || !Array.isArray(results) || !results.length) return;
|
||||
const key = norm(q);
|
||||
const list = results.slice(0, PER_QUERY);
|
||||
const bytes = JSON.stringify(list).length;
|
||||
index[key] = { query: q, count: list.length, ts: ts || Date.now(), bytes, thumb: (list[0] || {}).thumbnail || '' };
|
||||
idbPut({ key, query: q, results: list, ts: index[key].ts });
|
||||
const keys = Object.keys(index);
|
||||
if (keys.length > MAX_QUERIES) {
|
||||
const drop = keys.sort((a, b) => (index[a].ts || 0) - (index[b].ts || 0)).slice(0, keys.length - MAX_QUERIES);
|
||||
drop.forEach((k) => delete index[k]);
|
||||
idbDel(drop);
|
||||
}
|
||||
persistIndex();
|
||||
}
|
||||
return {
|
||||
put(q, results) {
|
||||
q = String(q || '').trim();
|
||||
if (!q || !Array.isArray(results) || !results.length) return;
|
||||
const o = read();
|
||||
o[norm(q)] = { query: q, results: results.slice(0, PER_QUERY), ts: Date.now() };
|
||||
const keys = Object.keys(o);
|
||||
if (keys.length > MAX_QUERIES) {
|
||||
keys.sort((a, b) => (o[a].ts || 0) - (o[b].ts || 0)).slice(0, keys.length - MAX_QUERIES).forEach((k) => delete o[k]);
|
||||
}
|
||||
write();
|
||||
FRESH_MS,
|
||||
PER_QUERY,
|
||||
put,
|
||||
// Promise of { query, results, ts } or null.
|
||||
async get(q) {
|
||||
const key = norm(q);
|
||||
if (!index[key]) return null;
|
||||
const rec = await idbGet(key);
|
||||
if (rec && Array.isArray(rec.results) && rec.results.length) return rec;
|
||||
delete index[key]; persistIndex(); // index entry without data
|
||||
return null;
|
||||
},
|
||||
get(q) { const h = read()[norm(q)]; return h && Array.isArray(h.results) && h.results.length ? h : null; },
|
||||
has(q) { return !!read()[norm(q)]; },
|
||||
// Newest first: [{ query, count, ts, thumb }]
|
||||
has(q) { return !!index[norm(q)]; },
|
||||
isFresh(ts) { return Date.now() - (ts || 0) < FRESH_MS; },
|
||||
// Newest first: [{ key, query, count, ts, thumb }]
|
||||
list() {
|
||||
const o = read();
|
||||
return Object.keys(o).map((k) => ({ key: k, query: o[k].query || k, count: (o[k].results || []).length, ts: o[k].ts || 0, thumb: ((o[k].results || [])[0] || {}).thumbnail || '' }))
|
||||
return Object.keys(index).map((k) => ({ key: k, query: index[k].query || k, count: index[k].count || 0, ts: index[k].ts || 0, thumb: index[k].thumb || '' }))
|
||||
.sort((a, b) => b.ts - a.ts);
|
||||
},
|
||||
remove(keys) { const o = read(); (keys || []).forEach((k) => delete o[k]); write(); },
|
||||
clear() { mem = {}; try { localStorage.removeItem(KEY); } catch { /* blocked */ } },
|
||||
bytes() { try { return (localStorage.getItem(KEY) || '').length; } catch { return 0; } },
|
||||
count() { return Object.keys(read()).length; },
|
||||
remove(keys) { (keys || []).forEach((k) => delete index[k]); idbDel(keys || []); persistIndex(); },
|
||||
clear() { index = {}; idbClear(); persistIndex(); },
|
||||
bytes() { return Object.values(index).reduce((n, x) => n + (x.bytes || 0), 0); },
|
||||
count() { return Object.keys(index).length; },
|
||||
};
|
||||
})();
|
||||
|
||||
@@ -343,9 +389,13 @@ async function opfsClear() {
|
||||
let _currentBlobUrl = null;
|
||||
|
||||
const API = {
|
||||
search: (query) => WEB
|
||||
? webFetch(`/api/search?q=${encodeURIComponent(query)}`)
|
||||
search: (query, { refresh = false } = {}) => WEB
|
||||
? webFetch(`/api/search?q=${encodeURIComponent(query)}${refresh ? '&refresh=1' : ''}`)
|
||||
: call('yt.search', 'yt_search', { query }),
|
||||
// Videos the server already knows — answers fast while the real search loads.
|
||||
searchLocal: (query) => WEB
|
||||
? webFetch(`/api/search/local?q=${encodeURIComponent(query)}`)
|
||||
: Promise.resolve({ ok: false }),
|
||||
getChannel: (channel) => WEB
|
||||
? webFetch(`/api/channel?c=${encodeURIComponent(channel)}`)
|
||||
: call('yt.channel', 'yt_channel', { channel }),
|
||||
@@ -1303,7 +1353,7 @@ function startCacheResyncWatch() {
|
||||
// Pull a result set's artwork into the service worker's thumbnail cache, a few at
|
||||
// a time, so a saved search still has pictures offline.
|
||||
function warmSearchThumbs(results) {
|
||||
const urls = (results || []).map((r) => r && r.thumbnail).filter(Boolean).slice(0, 50);
|
||||
const urls = (results || []).map((r) => r && r.thumbnail).filter(Boolean).slice(0, 100);
|
||||
let i = 0;
|
||||
const next = () => { if (i >= urls.length) return; warmThumb(urls[i++]); setTimeout(next, 40); };
|
||||
next(); next(); next();
|
||||
@@ -7476,6 +7526,7 @@ function setListPaneHidden(hidden) {
|
||||
|
||||
function renderList() {
|
||||
{ const ls = $('listSub'); if (ls) ls.textContent = ''; const ps = $('p2pStrip'); if (ps) ps.classList.add('hidden'); }
|
||||
if (view.type !== 'search') { const sm = $('searchMeta'); if (sm) sm.classList.add('hidden'); }
|
||||
setListPaneHidden(view.type === 'search' && !searchResults.length && !searchAttempted);
|
||||
if (view.type === 'settings') { renderSettings(); return; }
|
||||
if (view.type === 'saved') { renderSaved(); return; }
|
||||
@@ -7878,7 +7929,7 @@ function renderSavedSearches() {
|
||||
row.querySelector('.card-title').textContent = it.query;
|
||||
row.addEventListener('click', () => {
|
||||
els.searchInput.value = it.query;
|
||||
if (els.runSearch) els.runSearch(it.query, { instant: SearchLibrary.get(it.query) });
|
||||
if (els.runSearch) els.runSearch(it.query);
|
||||
});
|
||||
c.appendChild(row);
|
||||
});
|
||||
@@ -9535,7 +9586,11 @@ function wireUI() {
|
||||
// `searchSeq` guards against a slower response overwriting a newer one
|
||||
// (the instant paint above can itself be "in flight" behind a background
|
||||
// refresh when the user searches something else right after).
|
||||
async function runSearchQuery(q, { instant = null } = {}) {
|
||||
// The saved copy of `q` is read from IndexedDB. A copy younger than 15 days is
|
||||
// shown as is (no network); an older one is shown at once and refreshed behind
|
||||
// it. `refresh` (the "Update search" button) always asks the server for fresh
|
||||
// results, bypassing its caches too.
|
||||
async function runSearchQuery(q, { refresh = false } = {}) {
|
||||
q = (q || '').trim();
|
||||
if (!q) return;
|
||||
const mySeq = ++searchSeq;
|
||||
@@ -9543,10 +9598,16 @@ function wireUI() {
|
||||
RecentSearches.add(q);
|
||||
hideRecentSearches();
|
||||
view = { type: 'search' };
|
||||
const instant = refresh ? null : await RecentSearches.getResults(q);
|
||||
if (mySeq !== searchSeq) return;
|
||||
if (instant) {
|
||||
searchResults = instant.results;
|
||||
searchAttempted = true;
|
||||
render();
|
||||
showSearchMeta(q, instant.ts);
|
||||
// Even on a fresh copy, make sure thumbnails are in the SW cache.
|
||||
warmSearchThumbs(instant.results);
|
||||
if (SearchLibrary.isFresh(instant.ts)) { scrollListIntoViewPortrait(); return; }
|
||||
} else {
|
||||
render();
|
||||
showSearchSkeletons();
|
||||
@@ -9554,27 +9615,60 @@ function wireUI() {
|
||||
// Even when already on the search view (no view change for render() to
|
||||
// detect), a new query means the user wants to see the results.
|
||||
scrollListIntoViewPortrait();
|
||||
const metaBtn = $('searchUpdateBtn');
|
||||
if (metaBtn) { metaBtn.disabled = true; metaBtn.textContent = 'Updating…'; }
|
||||
let networkDone = false, paintedLocal = false;
|
||||
if (!instant && !refresh) {
|
||||
// Known videos from the server's metadata store, painted until YouTube answers.
|
||||
API.searchLocal(q).then((lr) => {
|
||||
if (networkDone || mySeq !== searchSeq || !lr || !lr.ok || !(lr.results || []).length) return;
|
||||
paintedLocal = true;
|
||||
searchResults = lr.results;
|
||||
searchAttempted = true;
|
||||
renderList();
|
||||
}).catch(() => {});
|
||||
}
|
||||
try {
|
||||
const res = await API.search(q);
|
||||
const res = await API.search(q, { refresh });
|
||||
networkDone = true;
|
||||
if (mySeq !== searchSeq) return; // superseded by a newer search
|
||||
if (!res || !res.ok) throw new Error(res?.error || 'Search failed');
|
||||
const results = res.results || [];
|
||||
searchResults = results;
|
||||
searchAttempted = true;
|
||||
renderList();
|
||||
RecentSearches.cacheResults(q, results);
|
||||
const ts = Date.now();
|
||||
RecentSearches.cacheResults(q, results, ts);
|
||||
showSearchMeta(q, ts);
|
||||
warmSearchThumbs(results);
|
||||
warmStreams(results.slice(0, 3).map((r) => r.id));
|
||||
if (refresh) toast('Search updated');
|
||||
perfMeasure('ytp:search', 'ytp:search-start');
|
||||
} catch (err) {
|
||||
networkDone = true;
|
||||
if (mySeq !== searchSeq) return;
|
||||
if (instant) return; // keep showing the cached results; fail silently
|
||||
if (paintedLocal) return; // keep the server's known videos
|
||||
if (instant) { showSearchMeta(q, instant.ts); if (refresh) toast('Could not update: ' + err.message); return; } // keep the saved copy
|
||||
els.cards.innerHTML = '';
|
||||
els.status.classList.remove('hidden');
|
||||
els.status.textContent = '⚠ ' + err.message;
|
||||
}
|
||||
}
|
||||
|
||||
// "Saved copy · 3 d ago [↻ Update search]" above the results.
|
||||
function showSearchMeta(q, ts) {
|
||||
const m = $('searchMeta');
|
||||
if (!m || view.type !== 'search' || !searchResults.length) return;
|
||||
const left = Math.max(0, Math.ceil((SearchLibrary.FRESH_MS - (Date.now() - ts)) / 86400000));
|
||||
m.innerHTML = `<span class="sm-info">Updated ${fmtAgo(ts)} · ${searchResults.length} results${SearchLibrary.isFresh(ts) ? ` · kept ${left} more day${left === 1 ? '' : 's'}` : ' · refreshing'}</span>`
|
||||
+ '<button type="button" id="searchUpdateBtn" class="sm-btn">↻ Update search</button>';
|
||||
m.classList.remove('hidden');
|
||||
m.dataset.q = q;
|
||||
}
|
||||
$('searchMeta').addEventListener('click', (e) => {
|
||||
if (e.target.closest('#searchUpdateBtn')) runSearchQuery($('searchMeta').dataset.q || '', { refresh: true });
|
||||
});
|
||||
|
||||
els.runSearch = runSearchQuery;
|
||||
els.searchForm.addEventListener('submit', (e) => {
|
||||
e.preventDefault();
|
||||
@@ -9635,7 +9729,7 @@ function wireUI() {
|
||||
e.preventDefault();
|
||||
const q = row.dataset.q;
|
||||
els.searchInput.value = q;
|
||||
runSearchQuery(q, { instant: RecentSearches.getResults(q) });
|
||||
runSearchQuery(q);
|
||||
}
|
||||
});
|
||||
document.addEventListener('click', (e) => {
|
||||
|
||||
Reference in New Issue
Block a user