133 lines
7.7 KiB
JavaScript
133 lines
7.7 KiB
JavaScript
// Reader view for watch-party presentations: fetch a public web page, pull
|
|
// out the article (Mozilla Readability) and return clean, safe HTML that every
|
|
// guest can read at their own font size.
|
|
//
|
|
// GET /api/party/:code/reader reads the link the party is presenting right now
|
|
//
|
|
// Only the party's current link can be fetched, so this is not an open proxy.
|
|
// The fetch refuses private/loopback addresses (also after every redirect),
|
|
// caps size and time, and the output keeps only an allow-list of tags and
|
|
// attributes.
|
|
import { lookup } from 'node:dns/promises';
|
|
import { isIP } from 'node:net';
|
|
import { parseHTML } from 'linkedom';
|
|
import { Readability } from '@mozilla/readability';
|
|
|
|
const MAX_BYTES = 3 * 1024 * 1024, TIMEOUT_MS = 12000, MAX_REDIRECTS = 4;
|
|
const TAGS = new Set(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol', 'li', 'blockquote', 'pre', 'code', 'em', 'strong', 'b', 'i', 'u', 's',
|
|
'a', 'img', 'figure', 'figcaption', 'br', 'hr', 'sup', 'sub', 'small', 'mark', 'table', 'thead', 'tbody', 'tr', 'th', 'td', 'caption', 'dl', 'dt', 'dd', 'span', 'div', 'section', 'article']);
|
|
const DROP = new Set(['script', 'style', 'iframe', 'object', 'embed', 'form', 'input', 'button', 'textarea', 'select', 'noscript', 'svg', 'math', 'video', 'audio', 'canvas', 'link', 'meta', 'base', 'template']);
|
|
|
|
export function privateAddress(ip) {
|
|
const v = isIP(ip);
|
|
if (v === 4) {
|
|
const [a, b] = ip.split('.').map(Number);
|
|
return a === 0 || a === 10 || a === 127 || (a === 100 && b >= 64 && b < 128) || (a === 169 && b === 254)
|
|
|| (a === 172 && b >= 16 && b < 32) || (a === 192 && b === 168) || (a === 198 && (b === 18 || b === 19)) || a >= 224;
|
|
}
|
|
if (v === 6) {
|
|
const x = ip.toLowerCase();
|
|
if (x.startsWith('::ffff:')) return privateAddress(x.slice(7));
|
|
return x === '::' || x === '::1' || x.startsWith('fc') || x.startsWith('fd') || x.startsWith('fe8') || x.startsWith('fe9')
|
|
|| x.startsWith('fea') || x.startsWith('feb') || x.startsWith('ff');
|
|
}
|
|
return true;
|
|
}
|
|
|
|
async function assertPublic(u, resolve) {
|
|
if (!/^https?:$/.test(u.protocol)) throw new Error('only http and https links can be read');
|
|
if (u.port && !['80', '443'].includes(u.port)) throw new Error('that port is not allowed');
|
|
if (u.username || u.password) throw new Error('links with credentials are not allowed');
|
|
const host = u.hostname.replace(/^\[|\]$/g, '');
|
|
const addrs = isIP(host) ? [host] : (await resolve(host)).map((a) => a.address);
|
|
if (!addrs.length || addrs.some(privateAddress)) throw new Error('that address is not on the public internet');
|
|
}
|
|
|
|
async function fetchPublic(url, { fetchImpl, resolve }) {
|
|
let u = new URL(url);
|
|
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
|
await assertPublic(u, resolve);
|
|
const res = await fetchImpl(u.href, { redirect: 'manual', signal: AbortSignal.timeout(TIMEOUT_MS),
|
|
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; YTPlayerReader/1.0)', Accept: 'text/html,application/xhtml+xml' } });
|
|
if (res.status >= 300 && res.status < 400 && res.headers.get('location')) { u = new URL(res.headers.get('location'), u); continue; }
|
|
if (!res.ok) throw new Error(`the site answered ${res.status}`);
|
|
const type = res.headers.get('content-type') || '';
|
|
if (!/html|xml/i.test(type)) throw new Error('that link is not a web page (try Original view)');
|
|
const reader = res.body.getReader(); const parts = []; let size = 0;
|
|
for (;;) {
|
|
const { done, value } = await reader.read(); if (done) break;
|
|
size += value.length; if (size > MAX_BYTES) { reader.cancel(); throw new Error('that page is too large to read'); }
|
|
parts.push(value);
|
|
}
|
|
const charset = (type.match(/charset=([\w-]+)/i) || [])[1] || 'utf-8';
|
|
let text; try { text = new TextDecoder(charset).decode(Buffer.concat(parts)); } catch { text = new TextDecoder().decode(Buffer.concat(parts)); }
|
|
return { html: text, finalUrl: u.href };
|
|
}
|
|
throw new Error('too many redirects');
|
|
}
|
|
|
|
// Rebuild the article from an allow-list; anything else is unwrapped or dropped.
|
|
export function sanitize(html, baseUrl) {
|
|
const { document } = parseHTML(`<!doctype html><html><body><div id="r">${html}</div></body></html>`);
|
|
const abs = (v) => { try { const x = new URL(v, baseUrl); return /^https?:$/.test(x.protocol) ? x.href : null; } catch { return null; } };
|
|
const walk = (node) => {
|
|
for (const child of [...node.childNodes]) {
|
|
if (child.nodeType === 8) { child.remove(); continue; }
|
|
if (child.nodeType !== 1) continue;
|
|
const tag = child.tagName.toLowerCase();
|
|
if (DROP.has(tag)) { child.remove(); continue; }
|
|
walk(child);
|
|
if (!TAGS.has(tag)) { child.replaceWith(...child.childNodes); continue; }
|
|
const keep = {};
|
|
if (tag === 'a') { const h = abs(child.getAttribute('href') || ''); if (h) Object.assign(keep, { href: h, target: '_blank', rel: 'noopener noreferrer nofollow' }); }
|
|
if (tag === 'img') {
|
|
const src = abs(child.getAttribute('src') || child.getAttribute('data-src') || '');
|
|
if (!src) { child.remove(); continue; }
|
|
Object.assign(keep, { src, alt: (child.getAttribute('alt') || '').slice(0, 300), loading: 'lazy', referrerpolicy: 'no-referrer' });
|
|
}
|
|
if ((tag === 'td' || tag === 'th') && /^\d{1,2}$/.test(child.getAttribute('colspan') || '')) keep.colspan = child.getAttribute('colspan');
|
|
for (const a of [...child.attributes]) child.removeAttribute(a.name);
|
|
for (const [k, v] of Object.entries(keep)) child.setAttribute(k, v);
|
|
}
|
|
};
|
|
const root = document.getElementById('r');
|
|
walk(root);
|
|
return root.innerHTML.trim();
|
|
}
|
|
|
|
export function extractArticle(html, url) {
|
|
const { document } = parseHTML(html);
|
|
const article = new Readability(document, { charThreshold: 300 }).parse();
|
|
if (!article || !article.content) throw new Error('could not find readable text on that page (try Original view)');
|
|
return {
|
|
title: String(article.title || '').trim().slice(0, 300), byline: String(article.byline || '').trim().slice(0, 200),
|
|
site: String(article.siteName || new URL(url).hostname.replace(/^www\./, '')).slice(0, 120),
|
|
excerpt: String(article.excerpt || '').slice(0, 500), content: sanitize(article.content, url),
|
|
length: Number(article.length) || 0,
|
|
};
|
|
}
|
|
|
|
export function registerReaderRoutes(app, { presentUrlOf, fetchImpl = fetch, resolve = (h) => lookup(h, { all: true }), now = Date.now }) {
|
|
const cache = new Map(); // url → { at, value }
|
|
const hits = new Map(); // ip → timestamps
|
|
const ipOf = (c) => (c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || '').split(',')[0].trim() || 'local';
|
|
app.get('/api/party/:code/reader', async (c) => {
|
|
const url = presentUrlOf(String(c.req.param('code') || '').toUpperCase());
|
|
if (!url) return c.json({ ok: false, error: 'this party is not presenting a link' }, 404);
|
|
const t = now(), ip = ipOf(c), mine = (hits.get(ip) || []).filter((x) => t - x < 60_000);
|
|
if (mine.length >= 30) return c.json({ ok: false, error: 'too many requests — try again in a minute' }, 429);
|
|
mine.push(t); hits.set(ip, mine); if (hits.size > 5000) hits.clear();
|
|
const cached = cache.get(url);
|
|
if (cached && t - cached.at < 10 * 60_000) return c.json({ ok: true, ...cached.value }, 200, { 'Cache-Control': 'no-store' });
|
|
try {
|
|
const page = await fetchPublic(url, { fetchImpl, resolve });
|
|
const value = { url, ...extractArticle(page.html, page.finalUrl) };
|
|
cache.set(url, { at: t, value });
|
|
if (cache.size > 60) cache.delete(cache.keys().next().value);
|
|
return c.json({ ok: true, ...value }, 200, { 'Cache-Control': 'no-store' });
|
|
} catch (err) {
|
|
return c.json({ ok: false, error: err.name === 'TimeoutError' ? 'the site took too long to answer' : err.message }, 502);
|
|
}
|
|
});
|
|
}
|