Compare commits
1 Commits
main
...
codex/rust
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
18e4459bab |
39
alt/bench.mjs
Normal file
39
alt/bench.mjs
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
const base = process.argv[2];
|
||||||
|
if (!base) { console.error('usage: node alt/bench.mjs <baseUrl> [--duration-ms=10000]'); process.exit(2); }
|
||||||
|
const duration = Number(process.argv.find(x => x.startsWith('--duration-ms='))?.split('=')[1]) || 10000;
|
||||||
|
const cachedId = process.env.TEST_MEDIA_ID || 'dQw4w9WgXcQ';
|
||||||
|
const query = encodeURIComponent(process.env.BENCH_QUERY || 'ytplayer-bench-seed');
|
||||||
|
const manifest = await (await fetch(new URL('/api/manifest', base))).json();
|
||||||
|
const cssHash = manifest.files?.['/styles.css']?.h;
|
||||||
|
const targets = [
|
||||||
|
['version', '/api/version'],
|
||||||
|
['static', `/styles.css${cssHash ? `?v=${cssHash}` : ''}`],
|
||||||
|
['cached-search', `/api/search?q=${query}`],
|
||||||
|
['media-range', `/api/media/${cachedId}?g=1`, { Range: 'bytes=0-1048575' }],
|
||||||
|
];
|
||||||
|
async function run(label, path, concurrency, headers = {}) {
|
||||||
|
let count = 0, responseBytes = 0, errors = 0;
|
||||||
|
const latencies = [];
|
||||||
|
const until = Date.now() + duration;
|
||||||
|
async function worker() {
|
||||||
|
while (Date.now() < until) {
|
||||||
|
const start = performance.now();
|
||||||
|
try {
|
||||||
|
const response = await fetch(new URL(path, base), { headers });
|
||||||
|
const body = await response.arrayBuffer();
|
||||||
|
responseBytes += body.byteLength;
|
||||||
|
if (!response.ok) errors++;
|
||||||
|
} catch { errors++; }
|
||||||
|
latencies.push(performance.now() - start);
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await Promise.all(Array.from({ length: concurrency }, worker));
|
||||||
|
latencies.sort((a, b) => a - b);
|
||||||
|
const pick = p => latencies[Math.min(latencies.length - 1, Math.floor(latencies.length * p))] || 0;
|
||||||
|
return { route: label, concurrency, durationMs: duration, requests: count, rps: +(count / (duration / 1000)).toFixed(1), responseBytes, errors, p50Ms: +pick(.5).toFixed(1), p95Ms: +pick(.95).toFixed(1), p99Ms: +pick(.99).toFixed(1) };
|
||||||
|
}
|
||||||
|
for (const concurrency of [1, 10, 50, 200]) {
|
||||||
|
for (const [label, path, headers] of targets) console.log(JSON.stringify(await run(label, path, concurrency, headers)));
|
||||||
|
}
|
||||||
71
alt/compare.mjs
Normal file
71
alt/compare.mjs
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
|
const [baseUrl, candidateUrl = process.env.RUST_URL || 'http://127.0.0.1:3000'] = process.argv.slice(2);
|
||||||
|
if (!baseUrl) {
|
||||||
|
console.error('usage: node alt/compare.mjs <bun-base-url> [rust-url]');
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
const mediaId = process.env.TEST_MEDIA_ID || 'dQw4w9WgXcQ';
|
||||||
|
const testQuery = encodeURIComponent(process.env.TEST_QUERY || 'ytplayer-bench-seed');
|
||||||
|
const results = [];
|
||||||
|
const shape = value => Array.isArray(value) ? value.map(shape) : value && typeof value === 'object'
|
||||||
|
? Object.fromEntries(Object.keys(value).sort().map(key => [key, shape(value[key])])) : typeof value;
|
||||||
|
const headerSet = response => ({
|
||||||
|
contentType: (response.headers.get('content-type') || '').split(';')[0],
|
||||||
|
cache: response.headers.get('cache-control'),
|
||||||
|
range: response.headers.get('content-range'),
|
||||||
|
acceptRanges: response.headers.get('accept-ranges'),
|
||||||
|
encoding: response.headers.get('content-encoding'),
|
||||||
|
assetHash: response.headers.get('x-asset-hash'),
|
||||||
|
etag: response.headers.get('etag'),
|
||||||
|
});
|
||||||
|
async function inspect(url, path, { method = 'GET', headers = {} } = {}) {
|
||||||
|
const response = await fetch(new URL(path, url), { method, headers });
|
||||||
|
const type = response.headers.get('content-type') || '';
|
||||||
|
let body = null;
|
||||||
|
if (type.includes('json')) body = await response.json();
|
||||||
|
else if (type.startsWith('text/')) body = await response.text();
|
||||||
|
else await response.arrayBuffer();
|
||||||
|
return { status: response.status, headers: headerSet(response), shape: body ? shape(body) : null, body };
|
||||||
|
}
|
||||||
|
function check(name, a, b, fields) {
|
||||||
|
const headersMatch = fields.every(field => a.headers[field] === b.headers[field]);
|
||||||
|
const compatible = a.status === b.status && JSON.stringify(a.shape) === JSON.stringify(b.shape) && headersMatch;
|
||||||
|
results.push({ name, compatible, base: { status: a.status, headers: a.headers, shape: a.shape }, candidate: { status: b.status, headers: b.headers, shape: b.shape } });
|
||||||
|
console.log(`${compatible ? 'PASS' : 'DIFF'} ${name}: ${a.status} / ${b.status}${headersMatch ? '' : ` headers ${JSON.stringify(a.headers)} / ${JSON.stringify(b.headers)}`}`);
|
||||||
|
}
|
||||||
|
const [baseManifest, candidateManifest] = await Promise.all([inspect(baseUrl, '/api/manifest'), inspect(candidateUrl, '/api/manifest')]);
|
||||||
|
check('manifest', baseManifest, candidateManifest, ['cache', 'etag', 'contentType']);
|
||||||
|
for (const [name, url, manifest] of [['base', baseUrl, baseManifest.body], ['candidate', candidateUrl, candidateManifest.body]]) {
|
||||||
|
const [index, sw] = await Promise.all([inspect(url, '/'), inspect(url, '/sw.js')]);
|
||||||
|
const htmlText = index.body;
|
||||||
|
const swText = sw.body;
|
||||||
|
const tagPattern = new RegExp(`<meta[^>]*name="ytp-build"[^>]*content="${manifest.buildTag}"`);
|
||||||
|
const stamped = index.status === 200 && tagPattern.test(htmlText) && (htmlText.includes(`styles.css?v=${manifest.files['/styles.css']?.h}`) || htmlText.includes(`styles.css?v=${manifest.buildTag}`));
|
||||||
|
const workerStamped = sw.status === 200 && swText.includes(`"${manifest.buildTag}"`) && !swText.includes("typeof __BUILD_TAG__ !== 'undefined'");
|
||||||
|
const compatible = stamped && workerStamped;
|
||||||
|
results.push({ name: `${name}-shell-stamping`, compatible, stamped, workerStamped });
|
||||||
|
console.log(`${compatible ? 'PASS' : 'DIFF'} ${name}-shell-stamping`);
|
||||||
|
}
|
||||||
|
const cssHash = baseManifest.body?.files?.['/styles.css']?.h || '';
|
||||||
|
const checks = [
|
||||||
|
['version', '/api/version'],
|
||||||
|
['static', `/styles.css?v=${cssHash}`],
|
||||||
|
['search-local', `/api/search/local?q=${testQuery}`],
|
||||||
|
['search', `/api/search?q=${testQuery}`],
|
||||||
|
['streams', `/api/streams?v=${mediaId}`],
|
||||||
|
['download-prepare', `/api/download/${mediaId}/prepare`],
|
||||||
|
['media-status', `/api/media/${mediaId}/status`],
|
||||||
|
];
|
||||||
|
for (const [name, path] of checks) {
|
||||||
|
const [base, candidate] = await Promise.all([inspect(baseUrl, path), inspect(candidateUrl, path)]);
|
||||||
|
const fields = name === 'static' ? ['contentType', 'cache', 'encoding', 'assetHash'] : ['contentType'];
|
||||||
|
check(name, base, candidate, fields);
|
||||||
|
}
|
||||||
|
const [baseRange, rustRange] = await Promise.all([
|
||||||
|
inspect(baseUrl, `/api/media/${mediaId}?g=1`, { headers: { Range: 'bytes=0-63' } }),
|
||||||
|
inspect(candidateUrl, `/api/media/${mediaId}?g=1`, { headers: { Range: 'bytes=0-63' } }),
|
||||||
|
]);
|
||||||
|
check('media-range', baseRange, rustRange, ['contentType', 'cache', 'range', 'acceptRanges', 'etag']);
|
||||||
|
if (results.some(result => !result.compatible)) process.exitCode = 1;
|
||||||
1
alt/rust/.gitignore
vendored
Normal file
1
alt/rust/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
|||||||
|
/target/
|
||||||
27
alt/rust/BENCH.md
Normal file
27
alt/rust/BENCH.md
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
# Local backend A/B benchmark
|
||||||
|
|
||||||
|
Measured 2026-10-14 on the same Linux host, Rust optimized release build vs Bun/Hono, loopback HTTP, Chromium not involved in the load generator. Both services served the identical `frontend/` tree and used a seeded SQLite search row plus a 1 MiB cached-media fixture. The benchmark ran sequentially, four routes, 750 ms per route/concurrency. `responseBytes` are Node fetch's decoded response-body bytes, not raw compressed wire bytes. These are short smoke measurements; use a longer run and production-like media/network before drawing capacity conclusions.
|
||||||
|
|
||||||
|
Command:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
BENCH_QUERY=rust-backend-parity-20261014 TEST_MEDIA_ID=dQw4w9WgXcQ node alt/bench.mjs http://127.0.0.1:3101 --duration-ms=750
|
||||||
|
BENCH_QUERY=rust-backend-parity-20261014 TEST_MEDIA_ID=dQw4w9WgXcQ node alt/bench.mjs http://127.0.0.1:3100 --duration-ms=750
|
||||||
|
```
|
||||||
|
|
||||||
|
Each cell is `requests/sec (p95 ms)`; all 32 route/concurrency runs reported zero errors.
|
||||||
|
|
||||||
|
| Concurrency | Backend | `/api/version` | CSS asset | Cached search | 1 MiB media range |
|
||||||
|
|---:|---|---:|---:|---:|---:|
|
||||||
|
| 1 | Bun | 2,425 (0.7) | 521 (3.4) | 1,041 (2.5) | 612 (3.5) |
|
||||||
|
| 1 | Rust release | 1,844 (0.9) | 665 (2.0) | 2,269 (0.6) | 480 (3.9) |
|
||||||
|
| 10 | Bun | 7,008 (2.7) | 1,179 (12.4) | 1,623 (17.2) | 685 (27.0) |
|
||||||
|
| 10 | Rust release | 3,872 (3.3) | 1,131 (12.9) | 4,228 (3.1) | 599 (27.4) |
|
||||||
|
| 50 | Bun | 10,033 (11.3) | 1,279 (44.3) | 1,783 (54.7) | 769 (105.5) |
|
||||||
|
| 50 | Rust release | 4,267 (17.4) | 1,199 (52.7) | 4,733 (14.5) | 689 (135.4) |
|
||||||
|
| 200 | Bun | 9,947 (33.7) | 1,333 (203.0) | 1,863 (161.0) | 1,029 (266.3) |
|
||||||
|
| 200 | Rust release | 4,396 (59.8) | 1,328 (202.7) | 4,533 (59.9) | 863 (644.7) |
|
||||||
|
|
||||||
|
The Rust backend led the cached-search route in this short run and was close on static CSS at high concurrency. Bun served `/api/version` and large media ranges faster at high concurrency; Rust's 1 MiB Range p95 widened to 644.7 ms at 200 connections. The shared in-memory static snapshot and boot-time Brotli/gzip encodings avoided disk lookup and per-request recompression, but the Range implementation still streams through Tokio file reads. No result implies production superiority.
|
||||||
|
|
||||||
|
Bun's `yt-dlp` Python worker could not import the system `yt_dlp` package on this host. Bench routes were seeded and do not exercise YouTube upstream latency or media encoding. Docker is unavailable on this host, so image build/compose smoke was not run.
|
||||||
1919
alt/rust/Cargo.lock
generated
Normal file
1919
alt/rust/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
25
alt/rust/Cargo.toml
Normal file
25
alt/rust/Cargo.toml
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
[package]
|
||||||
|
name = "ytplayer-rust-backend"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2021"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
axum = { version = "0.8", features = ["http1", "tokio", "query", "json"] }
|
||||||
|
base64 = "0.22"
|
||||||
|
brotli = "9"
|
||||||
|
bytes = "1"
|
||||||
|
flate2 = "1"
|
||||||
|
futures-util = "0.3"
|
||||||
|
hex = "0.4"
|
||||||
|
http = "1"
|
||||||
|
mime_guess = "2"
|
||||||
|
regex = "1"
|
||||||
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "stream", "gzip", "brotli"] }
|
||||||
|
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||||
|
serde = { version = "1", features = ["derive"] }
|
||||||
|
serde_json = { version = "1", features = ["preserve_order"] }
|
||||||
|
sha2 = "0.10"
|
||||||
|
tokio = { version = "1", features = ["full"] }
|
||||||
|
tokio-util = { version = "0.7", features = ["io"] }
|
||||||
|
tower-http = { version = "0.6", features = ["compression-full", "cors", "trace"] }
|
||||||
|
url = "2"
|
||||||
22
alt/rust/Dockerfile
Normal file
22
alt/rust/Dockerfile
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
# Build from the repository root: docker build -f alt/rust/Dockerfile -t ytplayer-rust .
|
||||||
|
FROM rust:1-bookworm AS builder
|
||||||
|
WORKDIR /build
|
||||||
|
COPY alt/rust/Cargo.toml alt/rust/Cargo.lock ./
|
||||||
|
COPY alt/rust/src ./src
|
||||||
|
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
|
--mount=type=cache,target=/usr/local/cargo/git \
|
||||||
|
--mount=type=cache,target=/build/target \
|
||||||
|
cargo build --release && cp target/release/ytplayer-rust-backend /tmp/ytplayer-rust-backend
|
||||||
|
|
||||||
|
FROM debian:bookworm-slim
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl ffmpeg python3 python3-pip \
|
||||||
|
&& pip3 install --break-system-packages --no-cache-dir yt-dlp \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
WORKDIR /app
|
||||||
|
COPY --from=builder /tmp/ytplayer-rust-backend /usr/local/bin/ytplayer-rust-backend
|
||||||
|
COPY frontend/ /app/public/
|
||||||
|
RUN mkdir -p /app/data/media
|
||||||
|
ENV PORT=3000 PUBLIC_DIR=/app/public DB_PATH=/app/data/ytplayer.db MEDIA_DIR=/app/data/media \
|
||||||
|
MEDIA_CACHE_MAX_BYTES=10737418240 YTDLP_PATH=/usr/local/bin/yt-dlp FFMPEG_PATH=/usr/bin/ffmpeg FFPROBE_PATH=/usr/bin/ffprobe
|
||||||
|
EXPOSE 3000
|
||||||
|
ENTRYPOINT ["/usr/local/bin/ytplayer-rust-backend"]
|
||||||
29
alt/rust/README.md
Normal file
29
alt/rust/README.md
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
# Rust backend experiment
|
||||||
|
|
||||||
|
Drop-in web backend experiment using Axum/Tokio, Tower HTTP compression, reqwest/rustls and SQLite. The existing `frontend/` is copied at image build time and is not modified.
|
||||||
|
|
||||||
|
## Run locally
|
||||||
|
|
||||||
|
From the repository root:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo run --manifest-path alt/rust/Cargo.toml
|
||||||
|
```
|
||||||
|
|
||||||
|
By default the service serves `./frontend` when `./public` does not exist. Set `PUBLIC_DIR`, `PORT`, `DB_PATH`, `MEDIA_DIR`, `MEDIA_VOLUME_MARKER`, `MEDIA_CACHE_MAX_BYTES`, `YTDLP_PATH`, `FFMPEG_PATH`, `FFPROBE_PATH`, `APP_VERSION`, `ASSET_HASHING`, and `ASSET_SYNC` as needed. `yt-dlp`, `ffmpeg` and `ffprobe` must be on PATH for live search/streams/media jobs.
|
||||||
|
|
||||||
|
The UI is available at `http://localhost:3000/`; `/api/version` is the health endpoint. Set `BUILD_TAG` only for the legacy `ASSET_HASHING=0` rollback mode. Normal mode computes the tag from runtime assets and stamps each CSS/JS URL.
|
||||||
|
|
||||||
|
## Container
|
||||||
|
|
||||||
|
From the repository root, `docker build -f alt/rust/Dockerfile -t ytplayer-rust .` or `docker compose -f alt/rust/docker-compose.yml up --build`. Compose exposes container port 3000 on external `dokploy-network` and persists SQLite/media data in `/app/data`; it intentionally has no host port mapping or proxy labels.
|
||||||
|
|
||||||
|
## Compare and benchmark
|
||||||
|
|
||||||
|
`node alt/compare.mjs <bun-base-url> [rust-url]` compares status, JSON shape, selected headers, static stamps, and byte-range responses. Set `TEST_MEDIA_ID` to a cached 11-character id for stream/download/range checks. `node alt/rust/browser-smoke.mjs <rust-url>` opens the actual UI in Chromium; set `SMOKE_QUERY` to a cached query to exercise search and card rendering. The benchmark command and measured local results are in [BENCH.md](BENCH.md).
|
||||||
|
|
||||||
|
## Implemented surface and known gaps
|
||||||
|
|
||||||
|
Static SPA delivery, asset manifest/version/SW stamping, compression, search with in-memory/SQLite cache + InnerTube and yt-dlp fallback, stream resolution/warming, YouTube range proxy, validated basic media caching, download preparation/range transfer, and local profile/user persistence are implemented. The notes/admin/P2P/party/remote/upload families and saved-profile password/key protection are intentionally not in this experimental build. Media caching deliberately skips HEVC optimization, advanced retry/backoff lanes, full LRU free-space guards, upload media, chapters, and transcode policy parity; use the production Bun backend for those requirements.
|
||||||
|
|
||||||
|
The cache job is detached from request lifetimes, but the current validation is a reduced safety gate (H.264 + AAC, complete demux, size/duration checks) and is not yet the production validation suite. This is an A/B prototype, not a production replacement.
|
||||||
26
alt/rust/browser-smoke.mjs
Normal file
26
alt/rust/browser-smoke.mjs
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { chromium } from 'playwright';
|
||||||
|
|
||||||
|
const base = process.argv[2] || 'http://127.0.0.1:3000';
|
||||||
|
const query = process.env.SMOKE_QUERY;
|
||||||
|
const browser = await chromium.launch({ headless: true });
|
||||||
|
const page = await browser.newPage({ viewport: { width: 1280, height: 800 } });
|
||||||
|
const pageErrors = [];
|
||||||
|
page.on('pageerror', error => pageErrors.push(error.message));
|
||||||
|
try {
|
||||||
|
const response = await page.goto(base, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
||||||
|
assert.equal(response?.status(), 200, 'index page should load');
|
||||||
|
assert.equal(await page.title(), 'YT Player');
|
||||||
|
assert.equal(await page.locator('meta[name="ytp-build"]').getAttribute('content'), (await (await fetch(new URL('/api/version', base))).json()).buildTag);
|
||||||
|
assert.equal(await page.locator('#searchInput').count(), 1, 'search control should mount');
|
||||||
|
if (query) {
|
||||||
|
await page.locator('#searchInput').fill(query);
|
||||||
|
await page.locator('#searchInput').press('Enter');
|
||||||
|
await page.waitForFunction(() => document.querySelectorAll('#cards .card').length > 0, { timeout: 15000 });
|
||||||
|
}
|
||||||
|
assert.deepEqual(pageErrors, [], 'page should not throw during boot or search');
|
||||||
|
console.log(JSON.stringify({ ok: true, url: base, title: await page.title(), searchCards: await page.locator('#cards .card').count(), pageErrors }));
|
||||||
|
} finally {
|
||||||
|
await browser.close();
|
||||||
|
}
|
||||||
32
alt/rust/docker-compose.yml
Normal file
32
alt/rust/docker-compose.yml
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
services:
|
||||||
|
app:
|
||||||
|
build:
|
||||||
|
context: ../..
|
||||||
|
dockerfile: alt/rust/Dockerfile
|
||||||
|
restart: unless-stopped
|
||||||
|
expose:
|
||||||
|
- "3000"
|
||||||
|
environment:
|
||||||
|
PORT: "3000"
|
||||||
|
DB_PATH: /app/data/ytplayer.db
|
||||||
|
MEDIA_DIR: /app/data/media
|
||||||
|
MEDIA_VOLUME_MARKER: ${MEDIA_VOLUME_MARKER:-}
|
||||||
|
MEDIA_CACHE_MAX_BYTES: ${MEDIA_CACHE_MAX_BYTES:-10737418240}
|
||||||
|
PUBLIC_DIR: /app/public
|
||||||
|
volumes:
|
||||||
|
- ytplayer-rust-data:/app/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:3000/api/version"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
start_period: 10s
|
||||||
|
networks:
|
||||||
|
- dokploy-network
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
ytplayer-rust-data:
|
||||||
|
|
||||||
|
networks:
|
||||||
|
dokploy-network:
|
||||||
|
external: true
|
||||||
1599
alt/rust/src/main.rs
Normal file
1599
alt/rust/src/main.rs
Normal file
File diff suppressed because it is too large
Load Diff
464
alt/rust/src/manifest.rs
Normal file
464
alt/rust/src/manifest.rs
Normal file
@@ -0,0 +1,464 @@
|
|||||||
|
use base64::{engine::general_purpose::STANDARD, Engine};
|
||||||
|
use bytes::Bytes;
|
||||||
|
use regex::Regex;
|
||||||
|
use serde_json::{json, Map, Value};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::{
|
||||||
|
collections::{BTreeMap, BTreeSet},
|
||||||
|
fs,
|
||||||
|
path::Path,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, serde::Serialize)]
|
||||||
|
pub struct AssetFile {
|
||||||
|
pub h: String,
|
||||||
|
pub s: usize,
|
||||||
|
pub g: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct AssetSnapshot {
|
||||||
|
pub build_tag: String,
|
||||||
|
pub manifest: Value,
|
||||||
|
pub delivery_files: BTreeMap<String, AssetFile>,
|
||||||
|
pub bytes: BTreeMap<String, Bytes>,
|
||||||
|
pub etags: BTreeMap<String, String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hash10(bytes: &[u8]) -> String {
|
||||||
|
hex::encode(Sha256::digest(bytes))[..10].to_string()
|
||||||
|
}
|
||||||
|
fn hash12(bytes: &[u8]) -> String {
|
||||||
|
hex::encode(Sha256::digest(bytes))[..12].to_string()
|
||||||
|
}
|
||||||
|
fn etag(bytes: &[u8]) -> String {
|
||||||
|
format!("\"{}\"", &hex::encode(Sha256::digest(bytes))[..32])
|
||||||
|
}
|
||||||
|
fn collect(dir: &Path, prefix: &str, out: &mut BTreeMap<String, Vec<u8>>) -> std::io::Result<()> {
|
||||||
|
let mut entries = fs::read_dir(dir)?.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
entries.sort_by_key(|e| e.file_name());
|
||||||
|
for entry in entries {
|
||||||
|
let path = entry.path();
|
||||||
|
let name = entry.file_name().to_string_lossy().to_string();
|
||||||
|
let url = format!("{prefix}/{name}");
|
||||||
|
if path.is_dir() {
|
||||||
|
collect(&path, &url, out)?;
|
||||||
|
} else {
|
||||||
|
out.insert(url, fs::read(path)?);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
fn runtime(path: &str, include: &BTreeSet<String>) -> bool {
|
||||||
|
if include.contains(path) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
path != "/admin.html"
|
||||||
|
&& !path.to_ascii_lowercase().split('/').any(|p| p == "perf")
|
||||||
|
&& ![".test.js", ".test.mjs", ".md", ".txt", ".c", ".entry.js"]
|
||||||
|
.iter()
|
||||||
|
.any(|s| path.to_ascii_lowercase().ends_with(s))
|
||||||
|
}
|
||||||
|
fn embed(
|
||||||
|
source: &str,
|
||||||
|
files: &BTreeMap<String, AssetFile>,
|
||||||
|
groups: &Value,
|
||||||
|
tag: &str,
|
||||||
|
asset_sync: bool,
|
||||||
|
hashing: bool,
|
||||||
|
) -> String {
|
||||||
|
let mut page_files = Map::new();
|
||||||
|
if let Some(groups_obj) = groups.as_object() {
|
||||||
|
for group in groups_obj.values() {
|
||||||
|
if let Some(paths) = group.get("files").and_then(Value::as_array) {
|
||||||
|
for p in paths.iter().filter_map(Value::as_str) {
|
||||||
|
if p == "/index.html" || p == "/sw.js" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some(file) = files.get(p) {
|
||||||
|
page_files.insert(p.to_string(), json!({"h":if hashing { file.h.as_str() } else { tag },"s":file.s,"g":file.g}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let assets = json!({"buildTag":tag,"groups":groups,"files":page_files,"appCache":asset_sync})
|
||||||
|
.to_string()
|
||||||
|
.replace('<', "\\u003c");
|
||||||
|
let re = Regex::new(r#"(?s)(<script type="application/json" id="ytp-assets">).*?(</script>)"#)
|
||||||
|
.unwrap();
|
||||||
|
re.replace(source, format!("$1{assets}$2")).into_owned()
|
||||||
|
}
|
||||||
|
fn stamp(source: &str, files: &BTreeMap<String, AssetFile>, tag: &str, hashing: bool) -> String {
|
||||||
|
if !hashing {
|
||||||
|
let tag_re = Regex::new(r"(?is)<(?:script|link)\b[^>]*>").unwrap();
|
||||||
|
let attr_re = Regex::new(r#"(?i)\b(href|src)=["']([^"']+\.(?:css|js))["']"#).unwrap();
|
||||||
|
return tag_re
|
||||||
|
.replace_all(
|
||||||
|
&source.replace("__BUILD_TAG__", tag),
|
||||||
|
|caps: ®ex::Captures| {
|
||||||
|
attr_re
|
||||||
|
.replace_all(&caps[0], |a: ®ex::Captures| {
|
||||||
|
let raw = &a[2];
|
||||||
|
if raw.starts_with('/') || raw.starts_with("//") || raw.contains(':') {
|
||||||
|
return a[0].to_string();
|
||||||
|
}
|
||||||
|
let separator = if raw.contains('?') { '&' } else { '?' };
|
||||||
|
format!("{}=\"{}{}v={}\"", &a[1], raw, separator, tag)
|
||||||
|
})
|
||||||
|
.into_owned()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.into_owned();
|
||||||
|
}
|
||||||
|
let tag_re = Regex::new(r"(?is)<(?:script|link)\b[^>]*>").unwrap();
|
||||||
|
let attr_re = Regex::new(r#"\b(href|src)=(?:"([^"]+)"|'([^']+)')"#).unwrap();
|
||||||
|
let stamped = tag_re
|
||||||
|
.replace_all(source, |caps: ®ex::Captures| {
|
||||||
|
let tag_text = caps.get(0).unwrap().as_str();
|
||||||
|
attr_re
|
||||||
|
.replace_all(tag_text, |a: ®ex::Captures| {
|
||||||
|
let raw = a.get(2).or_else(|| a.get(3)).unwrap().as_str();
|
||||||
|
if raw.contains(":") || raw.starts_with("//") || raw.starts_with('#') {
|
||||||
|
return a[0].to_string();
|
||||||
|
}
|
||||||
|
let url = url::Url::parse(&format!("https://assets.invalid/{raw}"));
|
||||||
|
let Ok(parsed) = url else {
|
||||||
|
return a[0].to_string();
|
||||||
|
};
|
||||||
|
let path = parsed.path();
|
||||||
|
let Some(file) = files.get(path) else {
|
||||||
|
return a[0].to_string();
|
||||||
|
};
|
||||||
|
if !(path.ends_with(".css") || path.ends_with(".js")) || path == "/sw.js" {
|
||||||
|
return a[0].to_string();
|
||||||
|
}
|
||||||
|
let base = raw.split(['?', '#']).next().unwrap_or(raw);
|
||||||
|
let suffix = if let Some((_, query)) = raw.split_once('?') {
|
||||||
|
let rest = query
|
||||||
|
.split('#')
|
||||||
|
.next()
|
||||||
|
.unwrap_or("")
|
||||||
|
.split('&')
|
||||||
|
.filter(|p| !p.starts_with("v="))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("&");
|
||||||
|
if rest.is_empty() {
|
||||||
|
format!("?v={}", file.h)
|
||||||
|
} else {
|
||||||
|
format!("?v={}&{}", file.h, rest)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
format!("?v={}", file.h)
|
||||||
|
};
|
||||||
|
let frag = raw
|
||||||
|
.split_once('#')
|
||||||
|
.map(|(_, f)| format!("#{f}"))
|
||||||
|
.unwrap_or_default();
|
||||||
|
let quote = if a.get(2).is_some() { "\"" } else { "'" };
|
||||||
|
format!("{}={}{}{}{}{}", &a[1], quote, base, suffix, frag, quote)
|
||||||
|
})
|
||||||
|
.into_owned()
|
||||||
|
})
|
||||||
|
.into_owned();
|
||||||
|
stamped.replace("__BUILD_TAG__", tag)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn load(
|
||||||
|
root: &Path,
|
||||||
|
hashing: bool,
|
||||||
|
sync: bool,
|
||||||
|
tag_override: Option<&str>,
|
||||||
|
) -> Result<AssetSnapshot, Box<dyn std::error::Error>> {
|
||||||
|
let mut bytes = BTreeMap::new();
|
||||||
|
collect(root, "", &mut bytes)?;
|
||||||
|
let def: Value = bytes.get("/assets.json").map(|b| serde_json::from_slice(b)).transpose()?.unwrap_or_else(|| json!({"groups":{"core":{"contract":1,"eager":true,"files":bytes.keys().cloned().collect::<Vec<_>>()}}}));
|
||||||
|
let include = def
|
||||||
|
.get("include")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.filter_map(Value::as_str)
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect::<BTreeSet<_>>();
|
||||||
|
let mut groups = Map::new();
|
||||||
|
let mut contracts = Map::new();
|
||||||
|
let mut membership = BTreeMap::<String, String>::new();
|
||||||
|
if let Some(group_obj) = def.get("groups").and_then(Value::as_object) {
|
||||||
|
for name in group_obj.keys().cloned().collect::<BTreeSet<_>>() {
|
||||||
|
let group = group_obj[&name].clone();
|
||||||
|
let files = group
|
||||||
|
.get("files")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.ok_or("asset group files missing")?;
|
||||||
|
let mut uniq = BTreeSet::new();
|
||||||
|
for f in files.iter().filter_map(Value::as_str) {
|
||||||
|
if !runtime(f, &include) {
|
||||||
|
return Err(format!("excluded runtime file in group {name}: {f}").into());
|
||||||
|
}
|
||||||
|
if !bytes.contains_key(f) {
|
||||||
|
return Err(format!("missing asset group file {f}").into());
|
||||||
|
}
|
||||||
|
if !uniq.insert(f) || membership.insert(f.into(), name.clone()).is_some() {
|
||||||
|
return Err(format!("duplicate asset membership: {f}").into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
contracts.insert(
|
||||||
|
name.clone(),
|
||||||
|
group.get("contract").cloned().unwrap_or(json!(1)),
|
||||||
|
);
|
||||||
|
groups.insert(name, group);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let groups = Value::Object(groups);
|
||||||
|
let contracts = Value::Object(contracts);
|
||||||
|
let mut files = BTreeMap::new();
|
||||||
|
let mut delivery_files = BTreeMap::new();
|
||||||
|
for (path, content) in &bytes {
|
||||||
|
let f = AssetFile {
|
||||||
|
h: hash10(content),
|
||||||
|
s: content.len(),
|
||||||
|
g: membership
|
||||||
|
.get(path)
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(|| "core".into()),
|
||||||
|
};
|
||||||
|
delivery_files.insert(path.clone(), f.clone());
|
||||||
|
if runtime(path, &include) {
|
||||||
|
files.insert(path.clone(), f);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut source = String::from_utf8(bytes.get("/index.html").cloned().unwrap_or_default())?;
|
||||||
|
if let Some(app) = bytes.get("/app.js") {
|
||||||
|
source = source.replace("__APP_SCRIPT_HASH__", &STANDARD.encode(Sha256::digest(app)));
|
||||||
|
}
|
||||||
|
let sync = hashing && sync;
|
||||||
|
let embedded = embed(&source, &files, &groups, "__BUILD_TAG__", sync, hashing);
|
||||||
|
let canonical_index = stamp(&embedded, &files, "__BUILD_TAG__", hashing);
|
||||||
|
let mut canonical_files = Map::new();
|
||||||
|
for (path, file) in &files {
|
||||||
|
let mut obj = Map::new();
|
||||||
|
obj.insert("h".into(), json!(file.h));
|
||||||
|
obj.insert("s".into(), json!(file.s));
|
||||||
|
obj.insert("g".into(), json!(file.g));
|
||||||
|
canonical_files.insert(path.clone(), Value::Object(obj));
|
||||||
|
}
|
||||||
|
if hashing && files.contains_key("/index.html") {
|
||||||
|
let original = &files["/index.html"];
|
||||||
|
let mut obj = Map::new();
|
||||||
|
obj.insert("h".into(), json!(hash10(canonical_index.as_bytes())));
|
||||||
|
obj.insert("s".into(), json!(canonical_index.len()));
|
||||||
|
obj.insert("g".into(), json!(original.g));
|
||||||
|
obj.insert("source".into(), json!(original.h));
|
||||||
|
canonical_files.insert("/index.html".into(), Value::Object(obj));
|
||||||
|
}
|
||||||
|
let canonical =
|
||||||
|
json!({"assetSync":sync,"files":canonical_files,"groups":groups,"contracts":contracts});
|
||||||
|
let mut legacy_hasher = Sha256::new();
|
||||||
|
for (path, content) in &bytes {
|
||||||
|
if runtime(path, &include) {
|
||||||
|
legacy_hasher.update(format!("./public{path}").as_bytes());
|
||||||
|
legacy_hasher.update(content);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let legacy_tag = hex::encode(legacy_hasher.finalize())[..12].to_string();
|
||||||
|
let build_tag = if hashing {
|
||||||
|
hash12(serde_json::to_string(&canonical)?.as_bytes())
|
||||||
|
} else {
|
||||||
|
tag_override.unwrap_or(&legacy_tag).to_string()
|
||||||
|
};
|
||||||
|
let index = embed(
|
||||||
|
&stamp(&source, &files, &build_tag, hashing),
|
||||||
|
&files,
|
||||||
|
&groups,
|
||||||
|
&build_tag,
|
||||||
|
sync,
|
||||||
|
hashing,
|
||||||
|
);
|
||||||
|
let mut sw =
|
||||||
|
String::from_utf8(bytes.get("/sw.js").cloned().unwrap_or_default()).unwrap_or_default();
|
||||||
|
let inject =
|
||||||
|
Regex::new(r"typeof __BUILD_TAG__ !== 'undefined' \? __BUILD_TAG__ : '[^']*'").unwrap();
|
||||||
|
sw = inject.replace(&sw, format!("\"{build_tag}\"")).into_owned();
|
||||||
|
sw = sw.replace(
|
||||||
|
"typeof __ASSET_SYNC__ !== 'undefined' ? __ASSET_SYNC__ : true",
|
||||||
|
if sync { "true" } else { "false" },
|
||||||
|
);
|
||||||
|
if let Some(core) = files.get("/asset-sync-core.js") {
|
||||||
|
sw = sw.replace(
|
||||||
|
"importScripts('/asset-sync-core.js')",
|
||||||
|
&format!("importScripts('/asset-sync-core.js?v={}')", core.h),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if !index.is_empty() {
|
||||||
|
files.insert(
|
||||||
|
"/index.html".into(),
|
||||||
|
AssetFile {
|
||||||
|
h: hash10(index.as_bytes()),
|
||||||
|
s: index.len(),
|
||||||
|
g: "core".into(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if !sw.is_empty() {
|
||||||
|
files.insert(
|
||||||
|
"/sw.js".into(),
|
||||||
|
AssetFile {
|
||||||
|
h: hash10(sw.as_bytes()),
|
||||||
|
s: sw.len(),
|
||||||
|
g: "core".into(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if !index.is_empty() {
|
||||||
|
bytes.insert("/index.html".into(), index.as_bytes().to_vec());
|
||||||
|
}
|
||||||
|
if !sw.is_empty() {
|
||||||
|
bytes.insert("/sw.js".into(), sw.as_bytes().to_vec());
|
||||||
|
}
|
||||||
|
delivery_files.extend(files.clone());
|
||||||
|
let manifest =
|
||||||
|
json!({"buildTag":build_tag,"files":files,"groups":groups,"contracts":contracts});
|
||||||
|
let etags = bytes
|
||||||
|
.iter()
|
||||||
|
.map(|(path, content)| (path.clone(), etag(content)))
|
||||||
|
.collect();
|
||||||
|
let bytes = bytes
|
||||||
|
.into_iter()
|
||||||
|
.map(|(path, content)| (path, Bytes::from(content)))
|
||||||
|
.collect();
|
||||||
|
Ok(AssetSnapshot {
|
||||||
|
build_tag,
|
||||||
|
manifest,
|
||||||
|
delivery_files,
|
||||||
|
bytes,
|
||||||
|
etags,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cache_control(
|
||||||
|
path: &str,
|
||||||
|
version: Option<&str>,
|
||||||
|
current: Option<&AssetFile>,
|
||||||
|
hashing: bool,
|
||||||
|
build_tag: &str,
|
||||||
|
) -> &'static str {
|
||||||
|
if !hashing && path == "/index.html" {
|
||||||
|
return "no-cache";
|
||||||
|
}
|
||||||
|
if let Some(version) = version {
|
||||||
|
if current.map(|f| f.h.as_str()) == Some(version) || (!hashing && version == build_tag) {
|
||||||
|
return "public, max-age=31536000, immutable";
|
||||||
|
}
|
||||||
|
return if hashing { "no-store" } else { "no-cache" };
|
||||||
|
}
|
||||||
|
if path.starts_with("/fonts/") || path.starts_with("/icons/") {
|
||||||
|
"public, max-age=2592000"
|
||||||
|
} else {
|
||||||
|
"no-cache"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{cache_control, embed, stamp, AssetFile};
|
||||||
|
use std::{collections::BTreeMap, path::Path};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn repository_frontend_snapshot_has_hashed_assets_and_injected_tags() {
|
||||||
|
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frontend");
|
||||||
|
let snapshot = super::load(&root, true, true, None).expect("frontend asset manifest");
|
||||||
|
assert_eq!(snapshot.build_tag.len(), 12);
|
||||||
|
assert!(
|
||||||
|
snapshot.manifest["files"]["/app.js"]["h"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.len()
|
||||||
|
== 10
|
||||||
|
);
|
||||||
|
assert!(snapshot.bytes["/index.html"]
|
||||||
|
.windows(snapshot.build_tag.len())
|
||||||
|
.any(|w| w == snapshot.build_tag.as_bytes()));
|
||||||
|
let sw = String::from_utf8(snapshot.bytes["/sw.js"].to_vec()).unwrap();
|
||||||
|
assert!(sw.contains(&snapshot.build_tag));
|
||||||
|
let index = String::from_utf8(snapshot.bytes["/index.html"].to_vec()).unwrap();
|
||||||
|
assert!(index.contains(&format!(
|
||||||
|
"styles.css?v={}",
|
||||||
|
snapshot.manifest["files"]["/styles.css"]["h"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
)));
|
||||||
|
assert!(index.contains(&format!(
|
||||||
|
r#""/app.js":{{"h":"{}""#,
|
||||||
|
snapshot.manifest["files"]["/app.js"]["h"].as_str().unwrap()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rollback_snapshot_uses_legacy_single_tag() {
|
||||||
|
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frontend");
|
||||||
|
let snapshot =
|
||||||
|
super::load(&root, false, false, Some("legacy-build")).expect("rollback snapshot");
|
||||||
|
assert_eq!(snapshot.build_tag, "legacy-build");
|
||||||
|
let index = String::from_utf8(snapshot.bytes["/index.html"].to_vec()).unwrap();
|
||||||
|
assert!(index.contains("styles.css?v=legacy-build"));
|
||||||
|
assert!(index.contains(r#""/app.js":{"h":"legacy-build""#));
|
||||||
|
assert!(index.contains(r#""appCache":false"#));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_stamping_only_changes_relative_urls_in_resource_tags() {
|
||||||
|
let input = r#"<link rel="stylesheet" href="styles.css"><script src="/root.js"></script><script src="https://cdn.test/app.js"></script><a href="other.js">link</a>"#;
|
||||||
|
let output = stamp(input, &BTreeMap::new(), "legacy-tag", false);
|
||||||
|
assert!(output.contains(r#"href="styles.css?v=legacy-tag""#));
|
||||||
|
assert!(output.contains(r#"src="/root.js""#));
|
||||||
|
assert!(output.contains(r#"src="https://cdn.test/app.js""#));
|
||||||
|
assert!(output.contains(r#"href="other.js""#));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn legacy_index_embeds_single_build_tag_for_each_asset() {
|
||||||
|
let file = AssetFile {
|
||||||
|
h: "file-hash".into(),
|
||||||
|
s: 10,
|
||||||
|
g: "core".into(),
|
||||||
|
};
|
||||||
|
let files = BTreeMap::from([("/app.js".to_string(), file)]);
|
||||||
|
let groups = serde_json::json!({"core":{"files":["/app.js"]}});
|
||||||
|
let source = r#"<script type="application/json" id="ytp-assets">{}</script>"#;
|
||||||
|
let output = embed(source, &files, &groups, "legacy-tag", false, false);
|
||||||
|
assert!(output.contains(r#""h":"legacy-tag""#));
|
||||||
|
assert!(!output.contains("file-hash"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cache_header_matrix_uses_legacy_build_tag_and_file_hashes() {
|
||||||
|
let file = AssetFile {
|
||||||
|
h: "abc123".into(),
|
||||||
|
s: 10,
|
||||||
|
g: "core".into(),
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
cache_control(
|
||||||
|
"/app.js",
|
||||||
|
Some("legacy-tag"),
|
||||||
|
Some(&file),
|
||||||
|
false,
|
||||||
|
"legacy-tag"
|
||||||
|
),
|
||||||
|
"public, max-age=31536000, immutable"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
cache_control("/app.js", Some("old-tag"), Some(&file), false, "legacy-tag"),
|
||||||
|
"no-cache"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
cache_control("/app.js", Some("abc123"), Some(&file), true, "build"),
|
||||||
|
"public, max-age=31536000, immutable"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
cache_control("/app.js", Some("stale"), Some(&file), true, "build"),
|
||||||
|
"no-store"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user