Catalog collect: per-IP budget and fill-only writes; bulk import moves to an admin/API-token route

server/recommendations.js

@@ -77,7 +77,17 @@ export async function recommend(fingerprint = '', limit = 12) {
 
 // Capture cards from successful discovery responses, including future routes.
 // Incoming profile/sync payloads also carry cards absent from the response.
-export function registerCatalogRoutes(app, { resolveListener = async (c, name, fp) => name ? c.json({ ok: false, error: 'profile unavailable' }, 401) : fp } = {}) {
+export function registerCatalogRoutes(app, { resolveListener = async (c, name, fp) => name ? c.json({ ok: false, error: 'profile unavailable' }, 401) : fp,
+  adminAuth = async c => c.json({ ok: false, error: 'admin unavailable' }, 503) } = {}) {
+  // Anonymous submissions: a per-IP budget, and they may only add videos or fill blanks.
+  const collectLog = new Map();
+  const overCollectBudget = c => {
+    const ip = (c.req.header('cf-connecting-ip') || c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || '').split(',')[0].trim() || 'local';
+    const now = Date.now(), list = (collectLog.get(ip) || []).filter(t => now - t < 10 * 60_000);
+    list.push(now); collectLog.set(ip, list);
+    if (collectLog.size > 5000) collectLog.clear();
+    return list.length > 30;
+  };
   app.use('/api/*', async (c, next) => {
     const path = c.req.path;
     const incoming = c.req.method === 'POST' && /^\/api\/(user\/sync|profile\/(create|save)|playlist\/share)$/.test(path)
@@ -116,13 +126,26 @@ export function registerCatalogRoutes(app, { resolveListener = async (c, name, f
     }
     catch (e) { console.warn('[recommendations]', e.message); return c.json({ ok: false, error: 'Recommendations are unavailable right now.' }, 503); }
   });
-  app.post('/api/catalog/collect', async c => {
+  async function readCards(c, maxBytes, maxCards) {
-    if (Number(c.req.header('content-length')) > 512000) return c.json({ ok: false, error: 'metadata too large' }, 413);
+    if (Number(c.req.header('content-length')) > maxBytes) return c.json({ ok: false, error: 'metadata too large' }, 413);
     let body;
-    try { const raw = await c.req.text(); if (raw.length > 512000) return c.json({ ok: false, error: 'metadata too large' }, 413); body = JSON.parse(raw); }
+    try { const raw = await c.req.text(); if (raw.length > maxBytes) return c.json({ ok: false, error: 'metadata too large' }, 413); body = JSON.parse(raw); }
     catch { return c.json({ ok: false, error: 'invalid JSON' }, 400); }
-    if (!Array.isArray(body.cards) || body.cards.length > 500) return c.json({ ok: false, error: 'expected at most 500 cards' }, 400);
+    if (!Array.isArray(body?.cards) || body.cards.length > maxCards) return c.json({ ok: false, error: `expected at most ${maxCards} cards` }, 400);
-    try { return c.json({ ok: true, collected: await ingest(body.cards, 'client-search') }); }
+    return body.cards;
+  }
+  app.post('/api/catalog/collect', async c => {
+    if (overCollectBudget(c)) return c.json({ ok: false, error: 'too many submissions — try again later' }, 429);
+    const cards = await readCards(c, 512000, 500);
+    if (!Array.isArray(cards)) return cards;
+    try { return c.json({ ok: true, collected: await ingest(cards, 'client-search', { fillMissing: true }) }); }
+    catch { return c.json({ ok: false, error: 'Could not save metadata.' }, 503); }
+  });
+  // Bulk import (channel/playlist listings): admin login or API token; may overwrite metadata.
+  app.post('/api/admin/catalog/import', adminAuth, async c => {
+    const cards = await readCards(c, 4 * 1024 * 1024, 5000);
+    if (!Array.isArray(cards)) return cards;
+    try { return c.json({ ok: true, collected: await ingest(cards, 'collector') }); }
     catch { return c.json({ ok: false, error: 'Could not save metadata.' }, 503); }
   });
   app.get('/api/catalog/:id/thumbnail', async c => {

server/recommendations.test.js

@@ -130,6 +130,25 @@ test('client cached search collection validates input and stores new metadata',
   expect(picks.results).toHaveLength(4);
 });
 
+test('anonymous collection only fills blanks, is rate limited, and bulk import needs admin', async () => {
+  const deny = async c => c.json({ ok: false }, 401);
+  const allow = async (c, next) => c.req.header('x-test-admin') === 'yes' ? next() : deny(c);
+  const app = new Hono(); registerCatalogRoutes(app, { adminAuth: allow });
+  const card = { ...b, id: 'fillonlyvid', title: 'Original title' };
+  const ip = { 'X-Forwarded-For': '203.0.113.9' };
+  const post = (path, body, headers = {}) => app.request(path, { method: 'POST', headers: { 'Content-Type': 'application/json', ...headers }, body: JSON.stringify(body) });
+  expect((await post('/api/catalog/collect', { cards: [card] }, ip)).status).toBe(200);
+  await post('/api/catalog/collect', { cards: [{ ...card, title: 'Spam title' }] }, ip);
+  const title = async () => JSON.parse((await db.execute({ sql: 'SELECT card FROM video_meta WHERE id=?', args: [card.id] })).rows[0].card).title;
+  expect(await title()).toBe('Original title');
+  for (let i = 0; i < 28; i++) await post('/api/catalog/collect', { cards: [] }, ip);
+  expect((await post('/api/catalog/collect', { cards: [] }, ip)).status).toBe(429);
+  expect((await post('/api/catalog/collect', { cards: [] }, { 'X-Forwarded-For': '203.0.113.10' })).status).toBe(200);
+  expect((await post('/api/admin/catalog/import', { cards: [{ ...card, title: 'Fixed title' }] })).status).toBe(401);
+  expect(await (await post('/api/admin/catalog/import', { cards: [{ ...card, title: 'Fixed title' }] }, { 'x-test-admin': 'yes' })).json()).toEqual({ ok: true, collected: 1 });
+  expect(await title()).toBe('Fixed title');
+});
+
 test('legacy backfill progresses with a durable cursor and finds unplayed search metadata', async () => {
   const card = { ...b, id: 'legacyvideo' };
   await cache.put('legacy', [card]);

server/server.js

@@ -366,7 +366,8 @@ const requestMonitor = createRequestMonitor({
   },
 });
 if (process.env.OPS_MONITOR !== '0') app.use('*', requestMonitor.middleware());
-registerCatalogRoutes(app, { resolveListener: recommendationListener });
+registerCatalogRoutes(app, { resolveListener: recommendationListener,
+  adminAuth: (c, next) => notes.requireAdminOrToken(c, next) });  // notes is registered further down
 
 // ============================================================================
 // API routes