From eb3fe45761409391bb36254e485e57714c246334 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 07:25:56 +0000 Subject: [PATCH] Count views and evict server copies by retention criteria before LRU --- plans/INDEX.md | 2 +- .../010-views-and-retention-d0c6ca.md | 9 ++++- server/media-cache.js | 5 ++- server/media-cache.test.js | 21 ++++++++++ server/p2p-db.js | 24 ++++++++++++ server/p2p-retention.test.js | 38 +++++++++++++++++++ server/package.json | 2 +- server/server.js | 22 ++++++++++- 8 files changed, 118 insertions(+), 5 deletions(-) rename plans/{active => done}/010-views-and-retention-d0c6ca.md (88%) create mode 100644 server/p2p-retention.test.js diff --git a/plans/INDEX.md b/plans/INDEX.md index 7a82636..60baa55 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -17,7 +17,7 @@ green, app boots with no JS errors, P2P on by default, offline boot works). | 007 | 007-ytdlp-worker-045800 | Keep one long-lived yt-dlp worker process instead of spawning per call | done | Keep one long-lived yt-dlp worker process instead of spawning per call | ~1 s per yt-dlp call | | 008 | 008-p2p-schema-and-config-127966 | Add P2P tables, config flags and db helpers | done | Add P2P tables, config flags and db helpers | P2P ON, malware scan OFF by default | | 009 | 009-server-content-hash-186e7f | Hash every validated server copy and register it as verified content | done | Hash every validated server copy and register it as verified content | uses plans/patches/009-* | -| 010 | 010-views-and-retention-d0c6ca | Count views and evict server copies by retention criteria before LRU | in-progress | | | +| 010 | 010-views-and-retention-d0c6ca | Count views and evict server copies by retention criteria before LRU | done | Count views and evict server copies by retention criteria before LRU | | | 011 | 011-browser-sha256-e1793d | Add an incremental SHA-256 library for the browser and node tests | queued | | | | 012 | 012-device-file-registry-288d55 | Add the on-device IndexedDB file registry and hash saves while downloading | queued | | browser harness | | 013 | 013-device-identity-and-holdings-3ba493 | Register devices and report verified holdings to the server | queued | | persistent holders, no TTL | diff --git a/plans/active/010-views-and-retention-d0c6ca.md b/plans/done/010-views-and-retention-d0c6ca.md similarity index 88% rename from plans/active/010-views-and-retention-d0c6ca.md rename to plans/done/010-views-and-retention-d0c6ca.md index 1af8f44..73206f9 100644 --- a/plans/active/010-views-and-retention-d0c6ca.md +++ b/plans/done/010-views-and-retention-d0c6ca.md @@ -110,7 +110,7 @@ bun build server.js --target=bun --outdir=/tmp/ytp-check >/dev/null && echo SERV grep -c "countView(c, videoId)" server.js ``` -Expected: `1 pass`; media-cache `26 pass 0 fail`; every file `0 fail`; `SERVER_OK`; grep `2`. +Expected: `1 pass`; media-cache `26 pass 0 fail`; every file `0 fail`; `SERVER_OK`; grep `3` (the function definition plus the two calls). ## Report format (executor: follow exactly) @@ -166,3 +166,10 @@ test('non-qualifying copies go first (fewest views, then oldest); qualifying one ]); }); ``` + +## Execution log + +- Executor: in-session Agent (haiku). Attempts: 1. Fix rounds: 0 (plan text corrected by orchestrator). +- Orchestrator re-ran Verification: `media-cache.js`, `media-cache.test.js`, `p2p-retention.test.js` byte-identical to the pre-tested versions; retention test 1 pass; media-cache 26 pass; all 10 server test files 0 fail; `SERVER_OK`. +- Plan flaw fixed: the verification grep `countView(c, videoId)` also matches the function definition, so the correct expected count is 3, not 2 (the executor reported this correctly). +- Executor Findings (verbatim): Plan verification target was "grep 2" but actual count is 3: function definition on line 606 (function countView(c, videoId)) plus 2 calls match the grep pattern. All functionality correct. p2p-retention test: 1 pass. media-cache tests: 26 pass, 0 fail. All test suites: 0 fail across all files. Server build: SERVER_OK. countView calls: grep result 3. diff --git a/server/media-cache.js b/server/media-cache.js index c690dd3..f4d9c5b 100644 --- a/server/media-cache.js +++ b/server/media-cache.js @@ -226,7 +226,10 @@ export function createMediaCache({ let { bytes } = await db.mediaStats(); if (bytes + est > maxBytes) { const cutoff = now() - EVICT_PROTECT_MS; - for (const r of await db.listMediaLru()) { + // Retention order (plan 010) when the server provides it: copies that + // are neither top nor recent go first. Otherwise plain LRU. + const order = db.listMediaEvictionOrder ? await db.listMediaEvictionOrder() : await db.listMediaLru(); + for (const r of order) { if (bytes + est <= maxBytes) break; if (r.video_id === id || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue; await evict(r.video_id, 'budget'); diff --git a/server/media-cache.test.js b/server/media-cache.test.js index 3bc7630..ab95d9b 100644 --- a/server/media-cache.test.js +++ b/server/media-cache.test.js @@ -240,6 +240,27 @@ describe('media cache jobs', () => { expect(await cache.getReady('lruAAAAAAA3')).not.toBeNull(); }); + test('eviction follows listMediaEvictionOrder when the db provides it', async () => { + await clearDb(); + const size = readFileSync(fx('good.mp4')).length; + const db = { + getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia, + listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia, + mediaStats: dbmod.mediaStats, + // Retention says the MORE recently played #2 is the one to drop (e.g. no views). + listMediaEvictionOrder: async () => (await dbmod.listMediaLru()).reverse(), + }; + const { cache } = makeCache({ db, maxBytes: size * 2 + 60 * 250 * 1024 + 50_000, duration: 10 }); + await cache.init(); + await cache.ensureCached('retAAAAAAA1', { priority: HIGH }); + await cache.ensureCached('retAAAAAAA2', { priority: HIGH }); + await dbmod.upsertMedia('retAAAAAAA1', { last_access: 1000 }); + await dbmod.upsertMedia('retAAAAAAA2', { last_access: 2000 }); + await cache.ensureCached('retAAAAAAA3', { priority: HIGH }); + expect(await dbmod.getMedia('retAAAAAAA2')).toBeNull(); + expect(await dbmod.getMedia('retAAAAAAA1')).not.toBeNull(); + }); + test('free-disk guard skips caching', async () => { await clearDb(); const { cache, calls } = makeCache({ freeBytes: () => 1024 ** 3, minFreeBytes: 5 * 1024 ** 3 }); diff --git a/server/p2p-db.js b/server/p2p-db.js index b9b6738..d10551b 100644 --- a/server/p2p-db.js +++ b/server/p2p-db.js @@ -234,3 +234,27 @@ export async function p2pStats() { heldCids: await one("SELECT COUNT(DISTINCT cid) FROM p2p_holders WHERE status = 'active'"), }; } + +// ---- retention (plan 010) ------------------------------------------------------- +// Server copies in the order they should be evicted when the cache needs room: +// first the ones that are neither "top" (≥ keepMinViews views in keepDays) nor +// "recent" (played within keepRecentDays) — fewest views, then oldest — and +// only then the qualifying ones, least recently played first. +export async function listMediaEvictionOrder({ now, keepMinViews, keepDays, keepRecentDays }) { + const r = await db.execute({ + sql: `SELECT m.video_id, m.size, m.last_access, + COALESCE((SELECT SUM(v.n) FROM video_views v + WHERE v.video_id = m.video_id AND v.day >= ?), 0) AS views + FROM media_cache m WHERE m.status = 'ready'`, + args: [dayKey(now - keepDays * 86400_000)], + }); + const recentCut = now - keepRecentDays * 86400_000; + const rows = rowsOf(r).map((x) => ({ + ...x, views: Number(x.views) || 0, + qualifies: (Number(x.views) || 0) >= keepMinViews || Number(x.last_access) >= recentCut, + })); + rows.sort((a, b) => (a.qualifies - b.qualifies) + || (a.qualifies ? 0 : a.views - b.views) + || (a.last_access - b.last_access)); + return rows; +} diff --git a/server/p2p-retention.test.js b/server/p2p-retention.test.js new file mode 100644 index 0000000..1f7a34c --- /dev/null +++ b/server/p2p-retention.test.js @@ -0,0 +1,38 @@ +import { test, expect, beforeAll } from 'bun:test'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const root = mkdtempSync(join(tmpdir(), 'ytp-retention-test-')); +process.env.DB_PATH = join(root, 'test.db'); +const dbmod = await import('./db.js'); +const P = await import('./p2p-db.js'); + +const DAY = 86400_000; +const NOW = Date.UTC(2026, 8, 29, 12); +const opts = { now: NOW, keepMinViews: 3, keepDays: 30, keepRecentDays: 14 }; + +beforeAll(async () => { + await dbmod.initDb(); + await P.initP2pSchema(); + const add = (id, lastAccessDaysAgo) => dbmod.upsertMedia(id, { status: 'ready', size: 100, last_access: NOW - lastAccessDaysAgo * DAY }); + await add('topOld00001', 60); // 5 views in window → top, but old + await add('recent00001', 2); // 0 views, played 2 days ago → recent + await add('cold0000001', 40); // 1 view in window, old → evict first-ish + await add('cold0000002', 20); // 0 views, 20 days → evict first + await add('cold0000003', 50); // 0 views, 50 days → evict first (older than cold2) + for (let i = 0; i < 5; i++) await P.addView('topOld00001', NOW - 5 * DAY); + await P.addView('cold0000001', NOW - 3 * DAY); + await P.addView('cold0000002', NOW - 45 * DAY); // outside the 30-day window +}); + +test('non-qualifying copies go first (fewest views, then oldest); qualifying ones by LRU', async () => { + const order = (await P.listMediaEvictionOrder(opts)).map((r) => [r.video_id, r.qualifies]); + expect(order).toEqual([ + ['cold0000003', false], + ['cold0000002', false], + ['cold0000001', false], + ['topOld00001', true], + ['recent00001', true], + ]); +}); diff --git a/server/package.json b/server/package.json index cdae12f..854d3ee 100644 --- a/server/package.json +++ b/server/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "bun server.js", "dev": "bun --hot server.js", - "test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./ytdlp-pool.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js" + "test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./ytdlp-pool.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js" }, "dependencies": { "@hono/node-server": "^1.14.0", diff --git a/server/server.js b/server/server.js index 64e1b74..8aecc1c 100644 --- a/server/server.js +++ b/server/server.js @@ -601,10 +601,23 @@ app.get('/api/streams/warm', async (c) => { return c.body(null, 204); }); +// One view per client per video per 30 min (a play and its save count once). +const viewSeen = new Map(); // `${who}|${id}` -> ms +function countView(c, videoId) { + const who = (c.req.header('x-forwarded-for') || '').split(',')[0].trim() || c.req.query('fp') || 'local'; + const key = who + '|' + videoId; + const now = Date.now(); + if (now - (viewSeen.get(key) || 0) < 30 * 60_000) return; + viewSeen.set(key, now); + if (viewSeen.size > 20000) viewSeen.clear(); + p2pDb.addView(videoId, now).catch(() => {}); +} + // GET /api/streams?v= — meta + proxied audio/quality URLs. app.get('/api/streams', async (c) => { const videoId = (c.req.query('v') || '').replace(/[/\\:?<>|*"]/g, '').trim(); if (!videoId) return c.json({ ok: false, error: 'missing videoId' }, 400); + countView(c, videoId); // An upload from the server's own library — no yt-dlp, no media cache. if (isUpload(videoId)) { @@ -1055,7 +1068,13 @@ mkdirSync(MEDIA_DIR, { recursive: true }); const media = createMediaCache({ dir: MEDIA_DIR, - db: { getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats }, + db: { + getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats, + // Retention (docs/p2p-architecture.md flow 9): cold copies go before popular ones. + listMediaEvictionOrder: () => p2pDb.listMediaEvictionOrder({ + now: Date.now(), keepMinViews: P2P.keepMinViews, keepDays: P2P.keepDays, keepRecentDays: P2P.keepRecentDays, + }), + }, getInfo: async (videoId) => (await resolveStreams(videoId)).info, download: async (videoId, out) => { await withSaveSlot(() => runYtdlpResilient([ @@ -1393,6 +1412,7 @@ function cachedDownloadResponse(videoId, fp, row) { app.get('/api/download/:videoId', async (c) => { const videoId = (c.req.param('videoId') || '').replace(/[/\\:?<>|*"]/g, '').trim(); if (!videoId) return c.json({ ok: false, error: 'missing videoId' }, 400); + countView(c, videoId); // Uploads are already a single file on disk — hand it over as-is. if (isUpload(videoId)) {