josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Count views and evict server copies by retention criteria before LRU

Browse Source
This commit is contained in:
Claude
2026-09-30 07:25:56 +00:00
parent f12ca87131
commit eb3fe45761
8 changed files with 118 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
plans/INDEX.md
View File

@@ -17,7 +17,7 @@ green, app boots with no JS errors, P2P on by default, offline boot works).
| 007 | 007-ytdlp-worker-045800 | Keep one long-lived yt-dlp worker process instead of spawning per call | done | Keep one long-lived yt-dlp worker process instead of spawning per call | ~1 s per yt-dlp call |
| 008 | 008-p2p-schema-and-config-127966 | Add P2P tables, config flags and db helpers | done | Add P2P tables, config flags and db helpers | P2P ON, malware scan OFF by default |
| 009 | 009-server-content-hash-186e7f | Hash every validated server copy and register it as verified content | done | Hash every validated server copy and register it as verified content | uses plans/patches/009-* |
| 010 | 010-views-and-retention-d0c6ca | Count views and evict server copies by retention criteria before LRU | in-progress | | |
| 010 | 010-views-and-retention-d0c6ca | Count views and evict server copies by retention criteria before LRU | done | Count views and evict server copies by retention criteria before LRU | |
| 011 | 011-browser-sha256-e1793d | Add an incremental SHA-256 library for the browser and node tests | queued | | |
| 012 | 012-device-file-registry-288d55 | Add the on-device IndexedDB file registry and hash saves while downloading | queued | | browser harness |
| 013 | 013-device-identity-and-holdings-3ba493 | Register devices and report verified holdings to the server | queued | | persistent holders, no TTL |

9
plans/active/010-views-and-retention-d0c6ca.md → plans/done/010-views-and-retention-d0c6ca.md
View File

@@ -110,7 +110,7 @@ bun build server.js --target=bun --outdir=/tmp/ytp-check >/dev/null && echo SERV
grep -c "countView(c, videoId)" server.js
```
Expected: `1 pass`; media-cache `26 pass 0 fail`; every file `0 fail`; `SERVER_OK`; grep `2`.
Expected: `1 pass`; media-cache `26 pass 0 fail`; every file `0 fail`; `SERVER_OK`; grep `3` (the function definition plus the two calls).
## Report format (executor: follow exactly)
@@ -166,3 +166,10 @@ test('non-qualifying copies go first (fewest views, then oldest); qualifying one
]);
});
```
## Execution log
- Executor: in-session Agent (haiku). Attempts: 1. Fix rounds: 0 (plan text corrected by orchestrator).
- Orchestrator re-ran Verification: `media-cache.js`, `media-cache.test.js`, `p2p-retention.test.js` byte-identical to the pre-tested versions; retention test 1 pass; media-cache 26 pass; all 10 server test files 0 fail; `SERVER_OK`.
- Plan flaw fixed: the verification grep `countView(c, videoId)` also matches the function definition, so the correct expected count is 3, not 2 (the executor reported this correctly).
- Executor Findings (verbatim): Plan verification target was "grep 2" but actual count is 3: function definition on line 606 (function countView(c, videoId)) plus 2 calls match the grep pattern. All functionality correct. p2p-retention test: 1 pass. media-cache tests: 26 pass, 0 fail. All test suites: 0 fail across all files. Server build: SERVER_OK. countView calls: grep result 3.

5
server/media-cache.js
View File

@@ -226,7 +226,10 @@ export function createMediaCache({
let { bytes } = await db.mediaStats();
if (bytes + est > maxBytes) {
const cutoff = now() - EVICT_PROTECT_MS;
for (const r of await db.listMediaLru()) {
// Retention order (plan 010) when the server provides it: copies that
// are neither top nor recent go first. Otherwise plain LRU.
const order = db.listMediaEvictionOrder ? await db.listMediaEvictionOrder() : await db.listMediaLru();
for (const r of order) {
if (bytes + est <= maxBytes) break;
if (r.video_id === id || r.last_access > cutoff || jobs.has(r.video_id) || optActive === r.video_id) continue;
await evict(r.video_id, 'budget');

21
server/media-cache.test.js
View File

@@ -240,6 +240,27 @@ describe('media cache jobs', () => {
expect(await cache.getReady('lruAAAAAAA3')).not.toBeNull();
});
test('eviction follows listMediaEvictionOrder when the db provides it', async () => {
await clearDb();
const size = readFileSync(fx('good.mp4')).length;
const db = {
getMedia: dbmod.getMedia, upsertMedia: dbmod.upsertMedia, deleteMedia: dbmod.deleteMedia,
listMedia: dbmod.listMedia, listMediaLru: dbmod.listMediaLru, touchMedia: dbmod.touchMedia,
mediaStats: dbmod.mediaStats,
// Retention says the MORE recently played #2 is the one to drop (e.g. no views).
listMediaEvictionOrder: async () => (await dbmod.listMediaLru()).reverse(),
};
const { cache } = makeCache({ db, maxBytes: size * 2 + 60 * 250 * 1024 + 50_000, duration: 10 });
await cache.init();
await cache.ensureCached('retAAAAAAA1', { priority: HIGH });
await cache.ensureCached('retAAAAAAA2', { priority: HIGH });
await dbmod.upsertMedia('retAAAAAAA1', { last_access: 1000 });
await dbmod.upsertMedia('retAAAAAAA2', { last_access: 2000 });
await cache.ensureCached('retAAAAAAA3', { priority: HIGH });
expect(await dbmod.getMedia('retAAAAAAA2')).toBeNull();
expect(await dbmod.getMedia('retAAAAAAA1')).not.toBeNull();
});
test('free-disk guard skips caching', async () => {
await clearDb();
const { cache, calls } = makeCache({ freeBytes: () => 1024 ** 3, minFreeBytes: 5 * 1024 ** 3 });

24
server/p2p-db.js
View File

@@ -234,3 +234,27 @@ export async function p2pStats() {
heldCids: await one("SELECT COUNT(DISTINCT cid) FROM p2p_holders WHERE status = 'active'"),
};
}
// ---- retention (plan 010) -------------------------------------------------------
// Server copies in the order they should be evicted when the cache needs room:
// first the ones that are neither "top" (≥ keepMinViews views in keepDays) nor
// "recent" (played within keepRecentDays) — fewest views, then oldest — and
// only then the qualifying ones, least recently played first.
export async function listMediaEvictionOrder({ now, keepMinViews, keepDays, keepRecentDays }) {
const r = await db.execute({
sql: `SELECT m.video_id, m.size, m.last_access,
COALESCE((SELECT SUM(v.n) FROM video_views v
WHERE v.video_id = m.video_id AND v.day >= ?), 0) AS views
FROM media_cache m WHERE m.status = 'ready'`,
args: [dayKey(now - keepDays * 86400_000)],
});
const recentCut = now - keepRecentDays * 86400_000;
const rows = rowsOf(r).map((x) => ({
...x, views: Number(x.views) || 0,
qualifies: (Number(x.views) || 0) >= keepMinViews || Number(x.last_access) >= recentCut,
}));
rows.sort((a, b) => (a.qualifies - b.qualifies)
|| (a.qualifies ? 0 : a.views - b.views)
|| (a.last_access - b.last_access));
return rows;
}

38
server/p2p-retention.test.js Normal file
View File

@@ -0,0 +1,38 @@
import { test, expect, beforeAll } from 'bun:test';
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
const root = mkdtempSync(join(tmpdir(), 'ytp-retention-test-'));
process.env.DB_PATH = join(root, 'test.db');
const dbmod = await import('./db.js');
const P = await import('./p2p-db.js');
const DAY = 86400_000;
const NOW = Date.UTC(2026, 8, 29, 12);
const opts = { now: NOW, keepMinViews: 3, keepDays: 30, keepRecentDays: 14 };
beforeAll(async () => {
await dbmod.initDb();
await P.initP2pSchema();
const add = (id, lastAccessDaysAgo) => dbmod.upsertMedia(id, { status: 'ready', size: 100, last_access: NOW - lastAccessDaysAgo * DAY });
await add('topOld00001', 60); // 5 views in window → top, but old
await add('recent00001', 2); // 0 views, played 2 days ago → recent
await add('cold0000001', 40); // 1 view in window, old → evict first-ish
await add('cold0000002', 20); // 0 views, 20 days → evict first
await add('cold0000003', 50); // 0 views, 50 days → evict first (older than cold2)
for (let i = 0; i < 5; i++) await P.addView('topOld00001', NOW - 5 * DAY);
await P.addView('cold0000001', NOW - 3 * DAY);
await P.addView('cold0000002', NOW - 45 * DAY); // outside the 30-day window
});
test('non-qualifying copies go first (fewest views, then oldest); qualifying ones by LRU', async () => {
const order = (await P.listMediaEvictionOrder(opts)).map((r) => [r.video_id, r.qualifies]);
expect(order).toEqual([
['cold0000003', false],
['cold0000002', false],
['cold0000001', false],
['topOld00001', true],
['recent00001', true],
]);
});

2
server/package.json
View File

@@ -6,7 +6,7 @@
"scripts": {
"start": "bun server.js",
"dev": "bun --hot server.js",
"test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./ytdlp-pool.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js"
"test": "bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./ytdlp-pool.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js"
},
"dependencies": {
"@hono/node-server": "^1.14.0",

22
server/server.js
View File

@@ -601,10 +601,23 @@ app.get('/api/streams/warm', async (c) => {
return c.body(null, 204);
});
// One view per client per video per 30 min (a play and its save count once).
const viewSeen = new Map(); // `${who}|${id}` -> ms
function countView(c, videoId) {
const who = (c.req.header('x-forwarded-for') || '').split(',')[0].trim() || c.req.query('fp') || 'local';
const key = who + '|' + videoId;
const now = Date.now();
if (now - (viewSeen.get(key) || 0) < 30 * 60_000) return;
viewSeen.set(key, now);
if (viewSeen.size > 20000) viewSeen.clear();
p2pDb.addView(videoId, now).catch(() => {});
}
// GET /api/streams?v=<videoId> — meta + proxied audio/quality URLs.
app.get('/api/streams', async (c) => {
const videoId = (c.req.query('v') || '').replace(/[/\\:?<>|*"]/g, '').trim();
if (!videoId) return c.json({ ok: false, error: 'missing videoId' }, 400);
countView(c, videoId);
// An upload from the server's own library — no yt-dlp, no media cache.
if (isUpload(videoId)) {
@@ -1055,7 +1068,13 @@ mkdirSync(MEDIA_DIR, { recursive: true });
const media = createMediaCache({
dir: MEDIA_DIR,
db: { getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats },
db: {
getMedia, upsertMedia, deleteMedia, listMedia, listMediaLru, touchMedia, mediaStats,
// Retention (docs/p2p-architecture.md flow 9): cold copies go before popular ones.
listMediaEvictionOrder: () => p2pDb.listMediaEvictionOrder({
now: Date.now(), keepMinViews: P2P.keepMinViews, keepDays: P2P.keepDays, keepRecentDays: P2P.keepRecentDays,
}),
},
getInfo: async (videoId) => (await resolveStreams(videoId)).info,
download: async (videoId, out) => {
await withSaveSlot(() => runYtdlpResilient([
@@ -1393,6 +1412,7 @@ function cachedDownloadResponse(videoId, fp, row) {
app.get('/api/download/:videoId', async (c) => {
const videoId = (c.req.param('videoId') || '').replace(/[/\\:?<>|*"]/g, '').trim();
if (!videoId) return c.json({ ok: false, error: 'missing videoId' }, 400);
countView(c, videoId);
// Uploads are already a single file on disk — hand it over as-is.
if (isUpload(videoId)) {