Watch party reader view: presented websites become clean text with per-device font size, landscape/portrait and an Original switch; SSRF-guarded server fetch
This commit is contained in:
@@ -7,7 +7,9 @@
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.14.0",
|
||||
"@libsql/client": "^0.14.0",
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"hono": "^4.7.10",
|
||||
"linkedom": "^0.18.13",
|
||||
"nodemailer": "^6",
|
||||
"qrcode": "^1.5.4",
|
||||
},
|
||||
@@ -40,6 +42,8 @@
|
||||
|
||||
"@libsql/win32-x64-msvc": ["@libsql/win32-x64-msvc@0.4.7", "", { "os": "win32", "cpu": "x64" }, "sha512-7pJzOWzPm6oJUxml+PCDRzYQ4A1hTMHAciTAHfFK4fkbDZX33nWPVG7Y3vqdKtslcwAzwmrNDc6sXy2nwWnbiw=="],
|
||||
|
||||
"@mozilla/readability": ["@mozilla/readability@0.6.0", "", {}, "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ=="],
|
||||
|
||||
"@neon-rs/load": ["@neon-rs/load@0.0.4", "", {}, "sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw=="],
|
||||
|
||||
"@types/node": ["@types/node@26.6.5", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg=="],
|
||||
@@ -50,6 +54,8 @@
|
||||
|
||||
"ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="],
|
||||
|
||||
"boolbase": ["boolbase@2.0.0", "", {}, "sha512-DkVaaQHymRhpYEYo9x1oo7Q7B0Y6KJUsjm3c9eTyFDby4MHLBTwZ6ZDWBel5zrYxj1WsZgC5oLpiz+93MluXeA=="],
|
||||
|
||||
"camelcase": ["camelcase@5.3.1", "", {}, "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg=="],
|
||||
|
||||
"cliui": ["cliui@6.0.0", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", "wrap-ansi": "^6.2.0" } }, "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ=="],
|
||||
@@ -58,6 +64,12 @@
|
||||
|
||||
"color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="],
|
||||
|
||||
"css-select": ["css-select@7.0.0", "", { "dependencies": { "boolbase": "^2.0.0", "css-what": "^8.0.0", "domhandler": "^6.0.1", "domutils": "^4.0.2", "nth-check": "^3.0.1" } }, "sha512-snmjEVXy+1LnwXdxhYvTMj1d9tOh4HxkA1YmoayVBeeyR2C14Pum7fcxJIm4SswYspVy866eYNwlH6xC3/VH5g=="],
|
||||
|
||||
"css-what": ["css-what@8.0.0", "", {}, "sha512-DH0Bqq3DNp5tdOReuNyAA+Ev4Y2GS5FMbZpeTLP6C4CDi0h5nL0BmUPChXw3o/qbHLDWHl49sbNqQVY7bMSDdw=="],
|
||||
|
||||
"cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="],
|
||||
|
||||
"data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="],
|
||||
|
||||
"decamelize": ["decamelize@1.2.0", "", {}, "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA=="],
|
||||
@@ -66,8 +78,18 @@
|
||||
|
||||
"dijkstrajs": ["dijkstrajs@1.0.3", "", {}, "sha512-qiSlmBq9+BCdCA/L46dw8Uy93mloxsPSbwnm5yrKn2vMPiy8KyAskTF6zuV/j5BMsmOGZDPs7KjU+mjb670kfA=="],
|
||||
|
||||
"dom-serializer": ["dom-serializer@3.1.1", "", { "dependencies": { "domelementtype": "^3.0.0", "domhandler": "^6.0.0", "entities": "^8.0.0" } }, "sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw=="],
|
||||
|
||||
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
|
||||
|
||||
"domhandler": ["domhandler@6.0.1", "", { "dependencies": { "domelementtype": "^3.0.0" } }, "sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg=="],
|
||||
|
||||
"domutils": ["domutils@4.0.2", "", { "dependencies": { "dom-serializer": "^3.0.0", "domelementtype": "^3.0.0", "domhandler": "^6.0.0" } }, "sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA=="],
|
||||
|
||||
"emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
|
||||
|
||||
"entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="],
|
||||
|
||||
"fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="],
|
||||
|
||||
"find-up": ["find-up@4.1.0", "", { "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" } }, "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw=="],
|
||||
@@ -78,12 +100,18 @@
|
||||
|
||||
"hono": ["hono@4.13.13", "", {}, "sha512-CQ46U0ZkAGmbT/4UxdzzGJpacP2IeKgY4a5/tOI9AABbpOMfK739wfDXmv1usCk+3RkKj1hQy4/fjhiwa2xlrA=="],
|
||||
|
||||
"html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="],
|
||||
|
||||
"htmlparser2": ["htmlparser2@10.1.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", "entities": "^7.0.1" } }, "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ=="],
|
||||
|
||||
"is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="],
|
||||
|
||||
"js-base64": ["js-base64@3.9.4", "", {}, "sha512-PtOMXpEGuP0RRiRXsjzHzl44dMHxSu2CPvAhinupR1tBa88me+1DPqsobl7eupn5UukjLkln1ZnAOAOXOrYG0Q=="],
|
||||
|
||||
"libsql": ["libsql@0.4.7", "", { "dependencies": { "@neon-rs/load": "^0.0.4", "detect-libc": "2.0.2" }, "optionalDependencies": { "@libsql/darwin-arm64": "0.4.7", "@libsql/darwin-x64": "0.4.7", "@libsql/linux-arm64-gnu": "0.4.7", "@libsql/linux-arm64-musl": "0.4.7", "@libsql/linux-x64-gnu": "0.4.7", "@libsql/linux-x64-musl": "0.4.7", "@libsql/win32-x64-msvc": "0.4.7" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "x64", "arm64", ] }, "sha512-T9eIRCs6b0J1SHKYIvD8+KCJMcWZ900iZyxdnSCdqxN12Z1ijzT+jY5nrk72Jw4B0HGzms2NgpryArlJqvc3Lw=="],
|
||||
|
||||
"linkedom": ["linkedom@0.18.13", "", { "dependencies": { "css-select": "^7.0.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", "htmlparser2": "^10.1.0", "uhyphen": "^0.2.0" }, "peerDependencies": { "canvas": ">= 2" }, "optionalPeers": ["canvas"] }, "sha512-ES/o9qotMpzpN2MHs+Iq/JcVoOj8Fa5wiQYrTdFpvAnwXL0g66XHHUc9WUMk6nAlBtGsFQ24ne+SYnvnaQ2FSw=="],
|
||||
|
||||
"locate-path": ["locate-path@5.0.0", "", { "dependencies": { "p-locate": "^4.1.0" } }, "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g=="],
|
||||
|
||||
"node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
|
||||
@@ -92,6 +120,8 @@
|
||||
|
||||
"nodemailer": ["nodemailer@6.10.1", "", {}, "sha512-Z+iLaBGVaSjbIzQ4pX6XV41HrooLsQ10ZWPUehGmuantvzWoDVBnmsdUcOIDM1t+yPor5pDhVlDESgOMEGxhHA=="],
|
||||
|
||||
"nth-check": ["nth-check@3.0.1", "", { "dependencies": { "boolbase": "^2.0.0" } }, "sha512-GX0gsdbGVCgnRgbeGaubfjpBXyYRWOOCVeYh08bSQvDZqxz5ndXs1OTfAt/h36G1xvI94YIspsI0sVFqAV9+RQ=="],
|
||||
|
||||
"p-limit": ["p-limit@2.3.0", "", { "dependencies": { "p-try": "^2.0.0" } }, "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w=="],
|
||||
|
||||
"p-locate": ["p-locate@4.1.0", "", { "dependencies": { "p-limit": "^2.2.0" } }, "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A=="],
|
||||
@@ -116,6 +146,8 @@
|
||||
|
||||
"strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
|
||||
|
||||
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
|
||||
|
||||
"undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="],
|
||||
|
||||
"web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="],
|
||||
@@ -131,5 +163,21 @@
|
||||
"yargs": ["yargs@15.4.1", "", { "dependencies": { "cliui": "^6.0.0", "decamelize": "^1.2.0", "find-up": "^4.1.0", "get-caller-file": "^2.0.1", "require-directory": "^2.1.1", "require-main-filename": "^2.0.0", "set-blocking": "^2.0.0", "string-width": "^4.2.0", "which-module": "^2.0.0", "y18n": "^4.0.0", "yargs-parser": "^18.1.2" } }, "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A=="],
|
||||
|
||||
"yargs-parser": ["yargs-parser@18.1.3", "", { "dependencies": { "camelcase": "^5.0.0", "decamelize": "^1.2.0" } }, "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ=="],
|
||||
|
||||
"dom-serializer/domelementtype": ["domelementtype@3.0.0", "", {}, "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg=="],
|
||||
|
||||
"dom-serializer/entities": ["entities@8.1.0", "", {}, "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA=="],
|
||||
|
||||
"domhandler/domelementtype": ["domelementtype@3.0.0", "", {}, "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg=="],
|
||||
|
||||
"domutils/domelementtype": ["domelementtype@3.0.0", "", {}, "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg=="],
|
||||
|
||||
"htmlparser2/domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="],
|
||||
|
||||
"htmlparser2/domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
||||
|
||||
"htmlparser2/domutils/dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
|
||||
|
||||
"htmlparser2/domutils/dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,12 +6,14 @@
|
||||
"scripts": {
|
||||
"start": "bun server.js",
|
||||
"dev": "bun --hot server.js",
|
||||
"test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./audit-log.test.js && bun test ./request-monitor.test.js && bun test ./admin-tools.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admin.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js && bun test ./profile-recovery.test.js"
|
||||
"test": "bun test ./recommendations.test.js && bun test --timeout 60000 ./media-cache.test.js && bun test ./notes.test.js && bun test ./audit-log.test.js && bun test ./request-monitor.test.js && bun test ./admin-tools.test.js && bun test ./transcriptions.test.js && bun test ./admin-analytics.test.js && bun test ./remote.test.js && bun test ./party.test.js && bun test ./uploads.test.js && bun test ./innertube.test.js && bun test ./related.test.js && bun test ./ytdlp-pool.test.js && bun test ./warm-queue.test.js && bun test ./p2p-db.test.js && bun test ./p2p-admin.test.js && bun test ./p2p-admit.test.js && bun test ./p2p-retention.test.js && bun test ./p2p-routes.test.js && bun test ./p2p-hub.test.js && bun test --timeout 60000 ./p2p-intake.test.js && bun test ./flags.test.js && bun test ./asset-manifest.test.js && bun test ./static-delivery.test.js && bun test ./video-catalog.test.js && bun test ./search-cache.test.js && bun test --timeout 60000 ./deep-search.test.js && bun test ./profile-recovery.test.js && bun test ./reader.test.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.14.0",
|
||||
"@libsql/client": "^0.14.0",
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"hono": "^4.7.10",
|
||||
"linkedom": "^0.18.13",
|
||||
"nodemailer": "^6",
|
||||
"qrcode": "^1.5.4"
|
||||
}
|
||||
|
||||
@@ -168,7 +168,7 @@ export function createPartyHub({ now = () => Date.now(), profileExists = async (
|
||||
else {
|
||||
const url = String(v?.url || '').trim();
|
||||
if (!/^https?:\/\/[^\s<>"]{3,2000}$/i.test(url)) return;
|
||||
p.present = { url, page: Math.max(1, Math.min(9999, Math.floor(Number(v.page) || 1))), by: me.name, at: now() };
|
||||
p.present = { url, mode: v.mode === 'reader' ? 'reader' : 'embed', page: Math.max(1, Math.min(9999, Math.floor(Number(v.page) || 1))), by: me.name, at: now() };
|
||||
}
|
||||
broadcast(p, { type: 'present', present: p.present });
|
||||
} else if ((m.type === 'invite' || m.type === 'uninvite') && isHost) {
|
||||
@@ -226,6 +226,7 @@ export function createPartyHub({ now = () => Date.now(), profileExists = async (
|
||||
return {
|
||||
upgrade,
|
||||
invitesFor,
|
||||
presentUrl: (code) => parties.get(code)?.present?.url || null,
|
||||
websocket: { maxPayloadLength: MAX_MSG, idleTimeout: 120, open, message: (ws, msg) => message(ws, typeof msg === 'string' ? msg : null), close },
|
||||
_parties: parties,
|
||||
stop() { clearInterval(sweeper); },
|
||||
|
||||
132
server/reader.js
Normal file
132
server/reader.js
Normal file
@@ -0,0 +1,132 @@
|
||||
// Reader view for watch-party presentations: fetch a public web page, pull
|
||||
// out the article (Mozilla Readability) and return clean, safe HTML that every
|
||||
// guest can read at their own font size.
|
||||
//
|
||||
// GET /api/party/:code/reader reads the link the party is presenting right now
|
||||
//
|
||||
// Only the party's current link can be fetched, so this is not an open proxy.
|
||||
// The fetch refuses private/loopback addresses (also after every redirect),
|
||||
// caps size and time, and the output keeps only an allow-list of tags and
|
||||
// attributes.
|
||||
import { lookup } from 'node:dns/promises';
|
||||
import { isIP } from 'node:net';
|
||||
import { parseHTML } from 'linkedom';
|
||||
import { Readability } from '@mozilla/readability';
|
||||
|
||||
const MAX_BYTES = 3 * 1024 * 1024, TIMEOUT_MS = 12000, MAX_REDIRECTS = 4;
|
||||
const TAGS = new Set(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol', 'li', 'blockquote', 'pre', 'code', 'em', 'strong', 'b', 'i', 'u', 's',
|
||||
'a', 'img', 'figure', 'figcaption', 'br', 'hr', 'sup', 'sub', 'small', 'mark', 'table', 'thead', 'tbody', 'tr', 'th', 'td', 'caption', 'dl', 'dt', 'dd', 'span', 'div', 'section', 'article']);
|
||||
const DROP = new Set(['script', 'style', 'iframe', 'object', 'embed', 'form', 'input', 'button', 'textarea', 'select', 'noscript', 'svg', 'math', 'video', 'audio', 'canvas', 'link', 'meta', 'base', 'template']);
|
||||
|
||||
export function privateAddress(ip) {
|
||||
const v = isIP(ip);
|
||||
if (v === 4) {
|
||||
const [a, b] = ip.split('.').map(Number);
|
||||
return a === 0 || a === 10 || a === 127 || (a === 100 && b >= 64 && b < 128) || (a === 169 && b === 254)
|
||||
|| (a === 172 && b >= 16 && b < 32) || (a === 192 && b === 168) || (a === 198 && (b === 18 || b === 19)) || a >= 224;
|
||||
}
|
||||
if (v === 6) {
|
||||
const x = ip.toLowerCase();
|
||||
if (x.startsWith('::ffff:')) return privateAddress(x.slice(7));
|
||||
return x === '::' || x === '::1' || x.startsWith('fc') || x.startsWith('fd') || x.startsWith('fe8') || x.startsWith('fe9')
|
||||
|| x.startsWith('fea') || x.startsWith('feb') || x.startsWith('ff');
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function assertPublic(u, resolve) {
|
||||
if (!/^https?:$/.test(u.protocol)) throw new Error('only http and https links can be read');
|
||||
if (u.port && !['80', '443'].includes(u.port)) throw new Error('that port is not allowed');
|
||||
if (u.username || u.password) throw new Error('links with credentials are not allowed');
|
||||
const host = u.hostname.replace(/^\[|\]$/g, '');
|
||||
const addrs = isIP(host) ? [host] : (await resolve(host)).map((a) => a.address);
|
||||
if (!addrs.length || addrs.some(privateAddress)) throw new Error('that address is not on the public internet');
|
||||
}
|
||||
|
||||
async function fetchPublic(url, { fetchImpl, resolve }) {
|
||||
let u = new URL(url);
|
||||
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
||||
await assertPublic(u, resolve);
|
||||
const res = await fetchImpl(u.href, { redirect: 'manual', signal: AbortSignal.timeout(TIMEOUT_MS),
|
||||
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; YTPlayerReader/1.0)', Accept: 'text/html,application/xhtml+xml' } });
|
||||
if (res.status >= 300 && res.status < 400 && res.headers.get('location')) { u = new URL(res.headers.get('location'), u); continue; }
|
||||
if (!res.ok) throw new Error(`the site answered ${res.status}`);
|
||||
const type = res.headers.get('content-type') || '';
|
||||
if (!/html|xml/i.test(type)) throw new Error('that link is not a web page (try Original view)');
|
||||
const reader = res.body.getReader(); const parts = []; let size = 0;
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read(); if (done) break;
|
||||
size += value.length; if (size > MAX_BYTES) { reader.cancel(); throw new Error('that page is too large to read'); }
|
||||
parts.push(value);
|
||||
}
|
||||
const charset = (type.match(/charset=([\w-]+)/i) || [])[1] || 'utf-8';
|
||||
let text; try { text = new TextDecoder(charset).decode(Buffer.concat(parts)); } catch { text = new TextDecoder().decode(Buffer.concat(parts)); }
|
||||
return { html: text, finalUrl: u.href };
|
||||
}
|
||||
throw new Error('too many redirects');
|
||||
}
|
||||
|
||||
// Rebuild the article from an allow-list; anything else is unwrapped or dropped.
|
||||
export function sanitize(html, baseUrl) {
|
||||
const { document } = parseHTML(`<!doctype html><html><body><div id="r">${html}</div></body></html>`);
|
||||
const abs = (v) => { try { const x = new URL(v, baseUrl); return /^https?:$/.test(x.protocol) ? x.href : null; } catch { return null; } };
|
||||
const walk = (node) => {
|
||||
for (const child of [...node.childNodes]) {
|
||||
if (child.nodeType === 8) { child.remove(); continue; }
|
||||
if (child.nodeType !== 1) continue;
|
||||
const tag = child.tagName.toLowerCase();
|
||||
if (DROP.has(tag)) { child.remove(); continue; }
|
||||
walk(child);
|
||||
if (!TAGS.has(tag)) { child.replaceWith(...child.childNodes); continue; }
|
||||
const keep = {};
|
||||
if (tag === 'a') { const h = abs(child.getAttribute('href') || ''); if (h) Object.assign(keep, { href: h, target: '_blank', rel: 'noopener noreferrer nofollow' }); }
|
||||
if (tag === 'img') {
|
||||
const src = abs(child.getAttribute('src') || child.getAttribute('data-src') || '');
|
||||
if (!src) { child.remove(); continue; }
|
||||
Object.assign(keep, { src, alt: (child.getAttribute('alt') || '').slice(0, 300), loading: 'lazy', referrerpolicy: 'no-referrer' });
|
||||
}
|
||||
if ((tag === 'td' || tag === 'th') && /^\d{1,2}$/.test(child.getAttribute('colspan') || '')) keep.colspan = child.getAttribute('colspan');
|
||||
for (const a of [...child.attributes]) child.removeAttribute(a.name);
|
||||
for (const [k, v] of Object.entries(keep)) child.setAttribute(k, v);
|
||||
}
|
||||
};
|
||||
const root = document.getElementById('r');
|
||||
walk(root);
|
||||
return root.innerHTML.trim();
|
||||
}
|
||||
|
||||
export function extractArticle(html, url) {
|
||||
const { document } = parseHTML(html);
|
||||
const article = new Readability(document, { charThreshold: 300 }).parse();
|
||||
if (!article || !article.content) throw new Error('could not find readable text on that page (try Original view)');
|
||||
return {
|
||||
title: String(article.title || '').trim().slice(0, 300), byline: String(article.byline || '').trim().slice(0, 200),
|
||||
site: String(article.siteName || new URL(url).hostname.replace(/^www\./, '')).slice(0, 120),
|
||||
excerpt: String(article.excerpt || '').slice(0, 500), content: sanitize(article.content, url),
|
||||
length: Number(article.length) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function registerReaderRoutes(app, { presentUrlOf, fetchImpl = fetch, resolve = (h) => lookup(h, { all: true }), now = Date.now }) {
|
||||
const cache = new Map(); // url → { at, value }
|
||||
const hits = new Map(); // ip → timestamps
|
||||
const ipOf = (c) => (c.req.header('x-forwarded-for') || c.req.header('x-real-ip') || '').split(',')[0].trim() || 'local';
|
||||
app.get('/api/party/:code/reader', async (c) => {
|
||||
const url = presentUrlOf(String(c.req.param('code') || '').toUpperCase());
|
||||
if (!url) return c.json({ ok: false, error: 'this party is not presenting a link' }, 404);
|
||||
const t = now(), ip = ipOf(c), mine = (hits.get(ip) || []).filter((x) => t - x < 60_000);
|
||||
if (mine.length >= 30) return c.json({ ok: false, error: 'too many requests — try again in a minute' }, 429);
|
||||
mine.push(t); hits.set(ip, mine); if (hits.size > 5000) hits.clear();
|
||||
const cached = cache.get(url);
|
||||
if (cached && t - cached.at < 10 * 60_000) return c.json({ ok: true, ...cached.value }, 200, { 'Cache-Control': 'no-store' });
|
||||
try {
|
||||
const page = await fetchPublic(url, { fetchImpl, resolve });
|
||||
const value = { url, ...extractArticle(page.html, page.finalUrl) };
|
||||
cache.set(url, { at: t, value });
|
||||
if (cache.size > 60) cache.delete(cache.keys().next().value);
|
||||
return c.json({ ok: true, ...value }, 200, { 'Cache-Control': 'no-store' });
|
||||
} catch (err) {
|
||||
return c.json({ ok: false, error: err.name === 'TimeoutError' ? 'the site took too long to answer' : err.message }, 502);
|
||||
}
|
||||
});
|
||||
}
|
||||
49
server/reader.test.js
Normal file
49
server/reader.test.js
Normal file
@@ -0,0 +1,49 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import { Hono } from 'hono';
|
||||
import { privateAddress, sanitize, extractArticle, registerReaderRoutes } from './reader.js';
|
||||
|
||||
const PAGE = `<!doctype html><html><head><title>Psalm 23 study</title></head><body>
|
||||
<nav>menu menu menu</nav>
|
||||
<article><h1>Psalm 23 study</h1><p class="by">By Ana</p>
|
||||
${'<p>The Lord is my shepherd; I shall not want. He makes me lie down in green pastures. He leads me beside still waters. </p>'.repeat(8)}
|
||||
<p>Read <a href="/next" onclick="steal()">the next part</a> <img src="/pic.jpg" onerror="x()"> <script>alert(1)</script></p>
|
||||
<iframe src="https://evil.example"></iframe><a href="javascript:alert(1)">bad</a></article></body></html>`;
|
||||
|
||||
test('private and special addresses are refused', () => {
|
||||
for (const ip of ['127.0.0.1', '10.8.0.1', '192.168.1.2', '172.20.0.1', '169.254.169.254', '100.64.0.1', '::1', 'fd00::1', '::ffff:10.0.0.1', '0.0.0.0'])
|
||||
expect(privateAddress(ip)).toBe(true);
|
||||
for (const ip of ['93.184.216.34', '1.1.1.1', '2606:4700::1111']) expect(privateAddress(ip)).toBe(false);
|
||||
});
|
||||
|
||||
test('sanitize keeps reading tags, makes links absolute and strips scripts/handlers', () => {
|
||||
const out = sanitize('<p style="x" onclick="a()">Hi <a href="/x">link</a><script>bad()</script><img src="i.png" onerror="y()"><iframe></iframe><custom>kept text</custom></p>', 'https://site.example/a/b');
|
||||
expect(out).toMatch(/<a [^>]*href="https:\/\/site\.example\/x"[^>]*>link<\/a>/);
|
||||
expect(out).toContain('rel="noopener noreferrer nofollow"');
|
||||
expect(out).toContain('src="https://site.example/a/i.png"');
|
||||
expect(out).toContain('kept text');
|
||||
for (const bad of ['script', 'onclick', 'onerror', 'iframe', 'style=', 'custom']) expect(out).not.toContain(bad);
|
||||
});
|
||||
|
||||
test('extractArticle finds the article and drops unsafe parts', () => {
|
||||
const a = extractArticle(PAGE, 'https://bible.example/psalm-23');
|
||||
expect(a.title).toBe('Psalm 23 study');
|
||||
expect(a.content).toContain('The Lord is my shepherd');
|
||||
expect(a.content).toContain('href="https://bible.example/next"');
|
||||
for (const bad of ['<script', 'onclick', 'onerror', 'javascript:', 'iframe']) expect(a.content).not.toContain(bad);
|
||||
});
|
||||
|
||||
test('the route reads only the presented link, refuses private targets and redirects into them', async () => {
|
||||
const present = { ABCDEF: 'https://bible.example/psalm-23', PRIVAT: 'http://10.0.0.5/admin', HOPPER: 'https://hop.example/' };
|
||||
const resolve = async (h) => [{ address: h === 'inside.example' ? '192.168.0.9' : '93.184.216.34' }];
|
||||
const fetchImpl = async (url) => url.startsWith('https://hop.example')
|
||||
? new Response('', { status: 302, headers: { location: 'http://inside.example/secret' } })
|
||||
: new Response(PAGE, { headers: { 'content-type': 'text/html; charset=utf-8' } });
|
||||
const app = new Hono();
|
||||
registerReaderRoutes(app, { presentUrlOf: (code) => present[code], fetchImpl, resolve });
|
||||
const get = async (code) => { const r = await app.request(`/api/party/${code}/reader`); return { status: r.status, json: await r.json() }; };
|
||||
expect((await get('ZZZZZZ')).status).toBe(404);
|
||||
const ok = await get('abcdef');
|
||||
expect(ok.json).toMatchObject({ ok: true, title: 'Psalm 23 study' });
|
||||
expect((await get('PRIVAT')).json.error).toMatch(/public internet/);
|
||||
expect((await get('HOPPER')).json.error).toMatch(/public internet/);
|
||||
});
|
||||
@@ -71,6 +71,7 @@ import { createYtdlpPool } from './ytdlp-pool.js';
|
||||
import { createWarmQueue } from './warm-queue.js';
|
||||
import { registerDeepSearchRoutes } from './deep-search.js';
|
||||
import { registerProfileRecoveryRoutes } from './profile-recovery.js';
|
||||
import { registerReaderRoutes } from './reader.js';
|
||||
import { dirname, join as pathJoin } from 'node:path';
|
||||
import { createRequestMonitor, registerRequestMonitorRoutes } from './request-monitor.js';
|
||||
import { registerP2pAdminRoutes } from './p2p-admin.js';
|
||||
@@ -2305,6 +2306,7 @@ app.get('/api/user/data', async (c) => {
|
||||
// ============================================================================
|
||||
const remote = createRemoteHub({ requireSameNetwork: process.env.REMOTE_SAME_NETWORK === '1' });
|
||||
const party = createPartyHub({ profileExists: async (name) => !!(await getProfile(name)) });
|
||||
registerReaderRoutes(app, { presentUrlOf: (code) => party.presentUrl(code) });
|
||||
// Running parties that invited this online profile (protected profiles need their secret).
|
||||
app.get('/api/party/invites', async (c) => {
|
||||
const name = (c.req.query('name') || '').trim().toLowerCase();
|
||||
|
||||
Reference in New Issue
Block a user