josh/ytplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Axum backend experiment

Browse Source
This commit is contained in:
Jonathan Sykes
2026-10-14 01:17:30 +08:00
parent f465cc86d0
commit 18e4459bab
12 changed files with 4254 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
alt/bench.mjs Normal file
View File

@@ -0,0 +1,39 @@
#!/usr/bin/env node
const base = process.argv[2];
if (!base) { console.error('usage: node alt/bench.mjs <baseUrl> [--duration-ms=10000]'); process.exit(2); }
const duration = Number(process.argv.find(x => x.startsWith('--duration-ms='))?.split('=')[1]) || 10000;
const cachedId = process.env.TEST_MEDIA_ID || 'dQw4w9WgXcQ';
const query = encodeURIComponent(process.env.BENCH_QUERY || 'ytplayer-bench-seed');
const manifest = await (await fetch(new URL('/api/manifest', base))).json();
const cssHash = manifest.files?.['/styles.css']?.h;
const targets = [
['version', '/api/version'],
['static', `/styles.css${cssHash ? `?v=${cssHash}` : ''}`],
['cached-search', `/api/search?q=${query}`],
['media-range', `/api/media/${cachedId}?g=1`, { Range: 'bytes=0-1048575' }],
];
async function run(label, path, concurrency, headers = {}) {
let count = 0, responseBytes = 0, errors = 0;
const latencies = [];
const until = Date.now() + duration;
async function worker() {
while (Date.now() < until) {
const start = performance.now();
try {
const response = await fetch(new URL(path, base), { headers });
const body = await response.arrayBuffer();
responseBytes += body.byteLength;
if (!response.ok) errors++;
} catch { errors++; }
latencies.push(performance.now() - start);
count++;
}
}
await Promise.all(Array.from({ length: concurrency }, worker));
latencies.sort((a, b) => a - b);
const pick = p => latencies[Math.min(latencies.length - 1, Math.floor(latencies.length * p))] || 0;
return { route: label, concurrency, durationMs: duration, requests: count, rps: +(count / (duration / 1000)).toFixed(1), responseBytes, errors, p50Ms: +pick(.5).toFixed(1), p95Ms: +pick(.95).toFixed(1), p99Ms: +pick(.99).toFixed(1) };
}
for (const concurrency of [1, 10, 50, 200]) {
for (const [label, path, headers] of targets) console.log(JSON.stringify(await run(label, path, concurrency, headers)));
}

71
alt/compare.mjs Normal file
View File

@@ -0,0 +1,71 @@
#!/usr/bin/env node
import assert from 'node:assert/strict';
const [baseUrl, candidateUrl = process.env.RUST_URL || 'http://127.0.0.1:3000'] = process.argv.slice(2);
if (!baseUrl) {
console.error('usage: node alt/compare.mjs <bun-base-url> [rust-url]');
process.exit(2);
}
const mediaId = process.env.TEST_MEDIA_ID || 'dQw4w9WgXcQ';
const testQuery = encodeURIComponent(process.env.TEST_QUERY || 'ytplayer-bench-seed');
const results = [];
const shape = value => Array.isArray(value) ? value.map(shape) : value && typeof value === 'object'
? Object.fromEntries(Object.keys(value).sort().map(key => [key, shape(value[key])])) : typeof value;
const headerSet = response => ({
contentType: (response.headers.get('content-type') || '').split(';')[0],
cache: response.headers.get('cache-control'),
range: response.headers.get('content-range'),
acceptRanges: response.headers.get('accept-ranges'),
encoding: response.headers.get('content-encoding'),
assetHash: response.headers.get('x-asset-hash'),
etag: response.headers.get('etag'),
});
async function inspect(url, path, { method = 'GET', headers = {} } = {}) {
const response = await fetch(new URL(path, url), { method, headers });
const type = response.headers.get('content-type') || '';
let body = null;
if (type.includes('json')) body = await response.json();
else if (type.startsWith('text/')) body = await response.text();
else await response.arrayBuffer();
return { status: response.status, headers: headerSet(response), shape: body ? shape(body) : null, body };
}
function check(name, a, b, fields) {
const headersMatch = fields.every(field => a.headers[field] === b.headers[field]);
const compatible = a.status === b.status && JSON.stringify(a.shape) === JSON.stringify(b.shape) && headersMatch;
results.push({ name, compatible, base: { status: a.status, headers: a.headers, shape: a.shape }, candidate: { status: b.status, headers: b.headers, shape: b.shape } });
console.log(`${compatible ? 'PASS' : 'DIFF'} ${name}: ${a.status} / ${b.status}${headersMatch ? '' : ` headers ${JSON.stringify(a.headers)} / ${JSON.stringify(b.headers)}`}`);
}
const [baseManifest, candidateManifest] = await Promise.all([inspect(baseUrl, '/api/manifest'), inspect(candidateUrl, '/api/manifest')]);
check('manifest', baseManifest, candidateManifest, ['cache', 'etag', 'contentType']);
for (const [name, url, manifest] of [['base', baseUrl, baseManifest.body], ['candidate', candidateUrl, candidateManifest.body]]) {
const [index, sw] = await Promise.all([inspect(url, '/'), inspect(url, '/sw.js')]);
const htmlText = index.body;
const swText = sw.body;
const tagPattern = new RegExp(`<meta[^>]*name="ytp-build"[^>]*content="${manifest.buildTag}"`);
const stamped = index.status === 200 && tagPattern.test(htmlText) && (htmlText.includes(`styles.css?v=${manifest.files['/styles.css']?.h}`) || htmlText.includes(`styles.css?v=${manifest.buildTag}`));
const workerStamped = sw.status === 200 && swText.includes(`"${manifest.buildTag}"`) && !swText.includes("typeof __BUILD_TAG__ !== 'undefined'");
const compatible = stamped && workerStamped;
results.push({ name: `${name}-shell-stamping`, compatible, stamped, workerStamped });
console.log(`${compatible ? 'PASS' : 'DIFF'} ${name}-shell-stamping`);
}
const cssHash = baseManifest.body?.files?.['/styles.css']?.h || '';
const checks = [
['version', '/api/version'],
['static', `/styles.css?v=${cssHash}`],
['search-local', `/api/search/local?q=${testQuery}`],
['search', `/api/search?q=${testQuery}`],
['streams', `/api/streams?v=${mediaId}`],
['download-prepare', `/api/download/${mediaId}/prepare`],
['media-status', `/api/media/${mediaId}/status`],
];
for (const [name, path] of checks) {
const [base, candidate] = await Promise.all([inspect(baseUrl, path), inspect(candidateUrl, path)]);
const fields = name === 'static' ? ['contentType', 'cache', 'encoding', 'assetHash'] : ['contentType'];
check(name, base, candidate, fields);
}
const [baseRange, rustRange] = await Promise.all([
inspect(baseUrl, `/api/media/${mediaId}?g=1`, { headers: { Range: 'bytes=0-63' } }),
inspect(candidateUrl, `/api/media/${mediaId}?g=1`, { headers: { Range: 'bytes=0-63' } }),
]);
check('media-range', baseRange, rustRange, ['contentType', 'cache', 'range', 'acceptRanges', 'etag']);
if (results.some(result => !result.compatible)) process.exitCode = 1;

1
alt/rust/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
/target/

27
alt/rust/BENCH.md Normal file
View File

@@ -0,0 +1,27 @@
# Local backend A/B benchmark
Measured 2026-10-14 on the same Linux host, Rust optimized release build vs Bun/Hono, loopback HTTP, Chromium not involved in the load generator. Both services served the identical `frontend/` tree and used a seeded SQLite search row plus a 1 MiB cached-media fixture. The benchmark ran sequentially, four routes, 750 ms per route/concurrency. `responseBytes` are Node fetch's decoded response-body bytes, not raw compressed wire bytes. These are short smoke measurements; use a longer run and production-like media/network before drawing capacity conclusions.
Command:
```sh
BENCH_QUERY=rust-backend-parity-20261014 TEST_MEDIA_ID=dQw4w9WgXcQ node alt/bench.mjs http://127.0.0.1:3101 --duration-ms=750
BENCH_QUERY=rust-backend-parity-20261014 TEST_MEDIA_ID=dQw4w9WgXcQ node alt/bench.mjs http://127.0.0.1:3100 --duration-ms=750
```
Each cell is `requests/sec (p95 ms)`; all 32 route/concurrency runs reported zero errors.
| Concurrency | Backend | `/api/version` | CSS asset | Cached search | 1 MiB media range |
|---:|---|---:|---:|---:|---:|
| 1 | Bun | 2,425 (0.7) | 521 (3.4) | 1,041 (2.5) | 612 (3.5) |
| 1 | Rust release | 1,844 (0.9) | 665 (2.0) | 2,269 (0.6) | 480 (3.9) |
| 10 | Bun | 7,008 (2.7) | 1,179 (12.4) | 1,623 (17.2) | 685 (27.0) |
| 10 | Rust release | 3,872 (3.3) | 1,131 (12.9) | 4,228 (3.1) | 599 (27.4) |
| 50 | Bun | 10,033 (11.3) | 1,279 (44.3) | 1,783 (54.7) | 769 (105.5) |
| 50 | Rust release | 4,267 (17.4) | 1,199 (52.7) | 4,733 (14.5) | 689 (135.4) |
| 200 | Bun | 9,947 (33.7) | 1,333 (203.0) | 1,863 (161.0) | 1,029 (266.3) |
| 200 | Rust release | 4,396 (59.8) | 1,328 (202.7) | 4,533 (59.9) | 863 (644.7) |
The Rust backend led the cached-search route in this short run and was close on static CSS at high concurrency. Bun served `/api/version` and large media ranges faster at high concurrency; Rust's 1 MiB Range p95 widened to 644.7 ms at 200 connections. The shared in-memory static snapshot and boot-time Brotli/gzip encodings avoided disk lookup and per-request recompression, but the Range implementation still streams through Tokio file reads. No result implies production superiority.
Bun's `yt-dlp` Python worker could not import the system `yt_dlp` package on this host. Bench routes were seeded and do not exercise YouTube upstream latency or media encoding. Docker is unavailable on this host, so image build/compose smoke was not run.

1919
alt/rust/Cargo.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

25
alt/rust/Cargo.toml Normal file
View File

@@ -0,0 +1,25 @@
[package]
name = "ytplayer-rust-backend"
version = "0.1.0"
edition = "2021"
[dependencies]
axum = { version = "0.8", features = ["http1", "tokio", "query", "json"] }
base64 = "0.22"
brotli = "9"
bytes = "1"
flate2 = "1"
futures-util = "0.3"
hex = "0.4"
http = "1"
mime_guess = "2"
regex = "1"
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "stream", "gzip", "brotli"] }
rusqlite = { version = "0.32", features = ["bundled"] }
serde = { version = "1", features = ["derive"] }
serde_json = { version = "1", features = ["preserve_order"] }
sha2 = "0.10"
tokio = { version = "1", features = ["full"] }
tokio-util = { version = "0.7", features = ["io"] }
tower-http = { version = "0.6", features = ["compression-full", "cors", "trace"] }
url = "2"

22
alt/rust/Dockerfile Normal file
View File

@@ -0,0 +1,22 @@
# Build from the repository root: docker build -f alt/rust/Dockerfile -t ytplayer-rust .
FROM rust:1-bookworm AS builder
WORKDIR /build
COPY alt/rust/Cargo.toml alt/rust/Cargo.lock ./
COPY alt/rust/src ./src
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/build/target \
cargo build --release && cp target/release/ytplayer-rust-backend /tmp/ytplayer-rust-backend
FROM debian:bookworm-slim
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl ffmpeg python3 python3-pip \
&& pip3 install --break-system-packages --no-cache-dir yt-dlp \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY --from=builder /tmp/ytplayer-rust-backend /usr/local/bin/ytplayer-rust-backend
COPY frontend/ /app/public/
RUN mkdir -p /app/data/media
ENV PORT=3000 PUBLIC_DIR=/app/public DB_PATH=/app/data/ytplayer.db MEDIA_DIR=/app/data/media \
MEDIA_CACHE_MAX_BYTES=10737418240 YTDLP_PATH=/usr/local/bin/yt-dlp FFMPEG_PATH=/usr/bin/ffmpeg FFPROBE_PATH=/usr/bin/ffprobe
EXPOSE 3000
ENTRYPOINT ["/usr/local/bin/ytplayer-rust-backend"]

29
alt/rust/README.md Normal file
View File

@@ -0,0 +1,29 @@
# Rust backend experiment
Drop-in web backend experiment using Axum/Tokio, Tower HTTP compression, reqwest/rustls and SQLite. The existing `frontend/` is copied at image build time and is not modified.
## Run locally
From the repository root:
```sh
cargo run --manifest-path alt/rust/Cargo.toml
```
By default the service serves `./frontend` when `./public` does not exist. Set `PUBLIC_DIR`, `PORT`, `DB_PATH`, `MEDIA_DIR`, `MEDIA_VOLUME_MARKER`, `MEDIA_CACHE_MAX_BYTES`, `YTDLP_PATH`, `FFMPEG_PATH`, `FFPROBE_PATH`, `APP_VERSION`, `ASSET_HASHING`, and `ASSET_SYNC` as needed. `yt-dlp`, `ffmpeg` and `ffprobe` must be on PATH for live search/streams/media jobs.
The UI is available at `http://localhost:3000/`; `/api/version` is the health endpoint. Set `BUILD_TAG` only for the legacy `ASSET_HASHING=0` rollback mode. Normal mode computes the tag from runtime assets and stamps each CSS/JS URL.
## Container
From the repository root, `docker build -f alt/rust/Dockerfile -t ytplayer-rust .` or `docker compose -f alt/rust/docker-compose.yml up --build`. Compose exposes container port 3000 on external `dokploy-network` and persists SQLite/media data in `/app/data`; it intentionally has no host port mapping or proxy labels.
## Compare and benchmark
`node alt/compare.mjs <bun-base-url> [rust-url]` compares status, JSON shape, selected headers, static stamps, and byte-range responses. Set `TEST_MEDIA_ID` to a cached 11-character id for stream/download/range checks. `node alt/rust/browser-smoke.mjs <rust-url>` opens the actual UI in Chromium; set `SMOKE_QUERY` to a cached query to exercise search and card rendering. The benchmark command and measured local results are in [BENCH.md](BENCH.md).
## Implemented surface and known gaps
Static SPA delivery, asset manifest/version/SW stamping, compression, search with in-memory/SQLite cache + InnerTube and yt-dlp fallback, stream resolution/warming, YouTube range proxy, validated basic media caching, download preparation/range transfer, and local profile/user persistence are implemented. The notes/admin/P2P/party/remote/upload families and saved-profile password/key protection are intentionally not in this experimental build. Media caching deliberately skips HEVC optimization, advanced retry/backoff lanes, full LRU free-space guards, upload media, chapters, and transcode policy parity; use the production Bun backend for those requirements.
The cache job is detached from request lifetimes, but the current validation is a reduced safety gate (H.264 + AAC, complete demux, size/duration checks) and is not yet the production validation suite. This is an A/B prototype, not a production replacement.

26
alt/rust/browser-smoke.mjs Normal file
View File

@@ -0,0 +1,26 @@
#!/usr/bin/env node
import assert from 'node:assert/strict';
import { chromium } from 'playwright';
const base = process.argv[2] || 'http://127.0.0.1:3000';
const query = process.env.SMOKE_QUERY;
const browser = await chromium.launch({ headless: true });
const page = await browser.newPage({ viewport: { width: 1280, height: 800 } });
const pageErrors = [];
page.on('pageerror', error => pageErrors.push(error.message));
try {
const response = await page.goto(base, { waitUntil: 'domcontentloaded', timeout: 30000 });
assert.equal(response?.status(), 200, 'index page should load');
assert.equal(await page.title(), 'YT Player');
assert.equal(await page.locator('meta[name="ytp-build"]').getAttribute('content'), (await (await fetch(new URL('/api/version', base))).json()).buildTag);
assert.equal(await page.locator('#searchInput').count(), 1, 'search control should mount');
if (query) {
await page.locator('#searchInput').fill(query);
await page.locator('#searchInput').press('Enter');
await page.waitForFunction(() => document.querySelectorAll('#cards .card').length > 0, { timeout: 15000 });
}
assert.deepEqual(pageErrors, [], 'page should not throw during boot or search');
console.log(JSON.stringify({ ok: true, url: base, title: await page.title(), searchCards: await page.locator('#cards .card').count(), pageErrors }));
} finally {
await browser.close();
}

32
alt/rust/docker-compose.yml Normal file
View File

@@ -0,0 +1,32 @@
services:
app:
build:
context: ../..
dockerfile: alt/rust/Dockerfile
restart: unless-stopped
expose:
- "3000"
environment:
PORT: "3000"
DB_PATH: /app/data/ytplayer.db
MEDIA_DIR: /app/data/media
MEDIA_VOLUME_MARKER: ${MEDIA_VOLUME_MARKER:-}
MEDIA_CACHE_MAX_BYTES: ${MEDIA_CACHE_MAX_BYTES:-10737418240}
PUBLIC_DIR: /app/public
volumes:
- ytplayer-rust-data:/app/data
healthcheck:
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:3000/api/version"]
interval: 30s
timeout: 5s
retries: 3
start_period: 10s
networks:
- dokploy-network
volumes:
ytplayer-rust-data:
networks:
dokploy-network:
external: true

1599
alt/rust/src/main.rs Normal file
View File

File diff suppressed because it is too large Load Diff

464
alt/rust/src/manifest.rs Normal file
View File

@@ -0,0 +1,464 @@
use base64::{engine::general_purpose::STANDARD, Engine};
use bytes::Bytes;
use regex::Regex;
use serde_json::{json, Map, Value};
use sha2::{Digest, Sha256};
use std::{
collections::{BTreeMap, BTreeSet},
fs,
path::Path,
};
#[derive(Clone, Debug, serde::Serialize)]
pub struct AssetFile {
pub h: String,
pub s: usize,
pub g: String,
}
#[derive(Clone)]
pub struct AssetSnapshot {
pub build_tag: String,
pub manifest: Value,
pub delivery_files: BTreeMap<String, AssetFile>,
pub bytes: BTreeMap<String, Bytes>,
pub etags: BTreeMap<String, String>,
}
fn hash10(bytes: &[u8]) -> String {
hex::encode(Sha256::digest(bytes))[..10].to_string()
}
fn hash12(bytes: &[u8]) -> String {
hex::encode(Sha256::digest(bytes))[..12].to_string()
}
fn etag(bytes: &[u8]) -> String {
format!("\"{}\"", &hex::encode(Sha256::digest(bytes))[..32])
}
fn collect(dir: &Path, prefix: &str, out: &mut BTreeMap<String, Vec<u8>>) -> std::io::Result<()> {
let mut entries = fs::read_dir(dir)?.collect::<Result<Vec<_>, _>>()?;
entries.sort_by_key(|e| e.file_name());
for entry in entries {
let path = entry.path();
let name = entry.file_name().to_string_lossy().to_string();
let url = format!("{prefix}/{name}");
if path.is_dir() {
collect(&path, &url, out)?;
} else {
out.insert(url, fs::read(path)?);
}
}
Ok(())
}
fn runtime(path: &str, include: &BTreeSet<String>) -> bool {
if include.contains(path) {
return true;
}
path != "/admin.html"
&& !path.to_ascii_lowercase().split('/').any(|p| p == "perf")
&& ![".test.js", ".test.mjs", ".md", ".txt", ".c", ".entry.js"]
.iter()
.any(|s| path.to_ascii_lowercase().ends_with(s))
}
fn embed(
source: &str,
files: &BTreeMap<String, AssetFile>,
groups: &Value,
tag: &str,
asset_sync: bool,
hashing: bool,
) -> String {
let mut page_files = Map::new();
if let Some(groups_obj) = groups.as_object() {
for group in groups_obj.values() {
if let Some(paths) = group.get("files").and_then(Value::as_array) {
for p in paths.iter().filter_map(Value::as_str) {
if p == "/index.html" || p == "/sw.js" {
continue;
}
if let Some(file) = files.get(p) {
page_files.insert(p.to_string(), json!({"h":if hashing { file.h.as_str() } else { tag },"s":file.s,"g":file.g}));
}
}
}
}
}
let assets = json!({"buildTag":tag,"groups":groups,"files":page_files,"appCache":asset_sync})
.to_string()
.replace('<', "\\u003c");
let re = Regex::new(r#"(?s)(<script type="application/json" id="ytp-assets">).*?(</script>)"#)
.unwrap();
re.replace(source, format!("$1{assets}$2")).into_owned()
}
fn stamp(source: &str, files: &BTreeMap<String, AssetFile>, tag: &str, hashing: bool) -> String {
if !hashing {
let tag_re = Regex::new(r"(?is)<(?:script|link)\b[^>]*>").unwrap();
let attr_re = Regex::new(r#"(?i)\b(href|src)=["']([^"']+\.(?:css|js))["']"#).unwrap();
return tag_re
.replace_all(
&source.replace("__BUILD_TAG__", tag),
|caps: &regex::Captures| {
attr_re
.replace_all(&caps[0], |a: &regex::Captures| {
let raw = &a[2];
if raw.starts_with('/') || raw.starts_with("//") || raw.contains(':') {
return a[0].to_string();
}
let separator = if raw.contains('?') { '&' } else { '?' };
format!("{}=\"{}{}v={}\"", &a[1], raw, separator, tag)
})
.into_owned()
},
)
.into_owned();
}
let tag_re = Regex::new(r"(?is)<(?:script|link)\b[^>]*>").unwrap();
let attr_re = Regex::new(r#"\b(href|src)=(?:"([^"]+)"|'([^']+)')"#).unwrap();
let stamped = tag_re
.replace_all(source, |caps: &regex::Captures| {
let tag_text = caps.get(0).unwrap().as_str();
attr_re
.replace_all(tag_text, |a: &regex::Captures| {
let raw = a.get(2).or_else(|| a.get(3)).unwrap().as_str();
if raw.contains(":") || raw.starts_with("//") || raw.starts_with('#') {
return a[0].to_string();
}
let url = url::Url::parse(&format!("https://assets.invalid/{raw}"));
let Ok(parsed) = url else {
return a[0].to_string();
};
let path = parsed.path();
let Some(file) = files.get(path) else {
return a[0].to_string();
};
if !(path.ends_with(".css") || path.ends_with(".js")) || path == "/sw.js" {
return a[0].to_string();
}
let base = raw.split(['?', '#']).next().unwrap_or(raw);
let suffix = if let Some((_, query)) = raw.split_once('?') {
let rest = query
.split('#')
.next()
.unwrap_or("")
.split('&')
.filter(|p| !p.starts_with("v="))
.collect::<Vec<_>>()
.join("&");
if rest.is_empty() {
format!("?v={}", file.h)
} else {
format!("?v={}&{}", file.h, rest)
}
} else {
format!("?v={}", file.h)
};
let frag = raw
.split_once('#')
.map(|(_, f)| format!("#{f}"))
.unwrap_or_default();
let quote = if a.get(2).is_some() { "\"" } else { "'" };
format!("{}={}{}{}{}{}", &a[1], quote, base, suffix, frag, quote)
})
.into_owned()
})
.into_owned();
stamped.replace("__BUILD_TAG__", tag)
}
pub fn load(
root: &Path,
hashing: bool,
sync: bool,
tag_override: Option<&str>,
) -> Result<AssetSnapshot, Box<dyn std::error::Error>> {
let mut bytes = BTreeMap::new();
collect(root, "", &mut bytes)?;
let def: Value = bytes.get("/assets.json").map(|b| serde_json::from_slice(b)).transpose()?.unwrap_or_else(|| json!({"groups":{"core":{"contract":1,"eager":true,"files":bytes.keys().cloned().collect::<Vec<_>>()}}}));
let include = def
.get("include")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_str)
.map(str::to_string)
.collect::<BTreeSet<_>>();
let mut groups = Map::new();
let mut contracts = Map::new();
let mut membership = BTreeMap::<String, String>::new();
if let Some(group_obj) = def.get("groups").and_then(Value::as_object) {
for name in group_obj.keys().cloned().collect::<BTreeSet<_>>() {
let group = group_obj[&name].clone();
let files = group
.get("files")
.and_then(Value::as_array)
.ok_or("asset group files missing")?;
let mut uniq = BTreeSet::new();
for f in files.iter().filter_map(Value::as_str) {
if !runtime(f, &include) {
return Err(format!("excluded runtime file in group {name}: {f}").into());
}
if !bytes.contains_key(f) {
return Err(format!("missing asset group file {f}").into());
}
if !uniq.insert(f) || membership.insert(f.into(), name.clone()).is_some() {
return Err(format!("duplicate asset membership: {f}").into());
}
}
contracts.insert(
name.clone(),
group.get("contract").cloned().unwrap_or(json!(1)),
);
groups.insert(name, group);
}
}
let groups = Value::Object(groups);
let contracts = Value::Object(contracts);
let mut files = BTreeMap::new();
let mut delivery_files = BTreeMap::new();
for (path, content) in &bytes {
let f = AssetFile {
h: hash10(content),
s: content.len(),
g: membership
.get(path)
.cloned()
.unwrap_or_else(|| "core".into()),
};
delivery_files.insert(path.clone(), f.clone());
if runtime(path, &include) {
files.insert(path.clone(), f);
}
}
let mut source = String::from_utf8(bytes.get("/index.html").cloned().unwrap_or_default())?;
if let Some(app) = bytes.get("/app.js") {
source = source.replace("__APP_SCRIPT_HASH__", &STANDARD.encode(Sha256::digest(app)));
}
let sync = hashing && sync;
let embedded = embed(&source, &files, &groups, "__BUILD_TAG__", sync, hashing);
let canonical_index = stamp(&embedded, &files, "__BUILD_TAG__", hashing);
let mut canonical_files = Map::new();
for (path, file) in &files {
let mut obj = Map::new();
obj.insert("h".into(), json!(file.h));
obj.insert("s".into(), json!(file.s));
obj.insert("g".into(), json!(file.g));
canonical_files.insert(path.clone(), Value::Object(obj));
}
if hashing && files.contains_key("/index.html") {
let original = &files["/index.html"];
let mut obj = Map::new();
obj.insert("h".into(), json!(hash10(canonical_index.as_bytes())));
obj.insert("s".into(), json!(canonical_index.len()));
obj.insert("g".into(), json!(original.g));
obj.insert("source".into(), json!(original.h));
canonical_files.insert("/index.html".into(), Value::Object(obj));
}
let canonical =
json!({"assetSync":sync,"files":canonical_files,"groups":groups,"contracts":contracts});
let mut legacy_hasher = Sha256::new();
for (path, content) in &bytes {
if runtime(path, &include) {
legacy_hasher.update(format!("./public{path}").as_bytes());
legacy_hasher.update(content);
}
}
let legacy_tag = hex::encode(legacy_hasher.finalize())[..12].to_string();
let build_tag = if hashing {
hash12(serde_json::to_string(&canonical)?.as_bytes())
} else {
tag_override.unwrap_or(&legacy_tag).to_string()
};
let index = embed(
&stamp(&source, &files, &build_tag, hashing),
&files,
&groups,
&build_tag,
sync,
hashing,
);
let mut sw =
String::from_utf8(bytes.get("/sw.js").cloned().unwrap_or_default()).unwrap_or_default();
let inject =
Regex::new(r"typeof __BUILD_TAG__ !== 'undefined' \? __BUILD_TAG__ : '[^']*'").unwrap();
sw = inject.replace(&sw, format!("\"{build_tag}\"")).into_owned();
sw = sw.replace(
"typeof __ASSET_SYNC__ !== 'undefined' ? __ASSET_SYNC__ : true",
if sync { "true" } else { "false" },
);
if let Some(core) = files.get("/asset-sync-core.js") {
sw = sw.replace(
"importScripts('/asset-sync-core.js')",
&format!("importScripts('/asset-sync-core.js?v={}')", core.h),
);
}
if !index.is_empty() {
files.insert(
"/index.html".into(),
AssetFile {
h: hash10(index.as_bytes()),
s: index.len(),
g: "core".into(),
},
);
}
if !sw.is_empty() {
files.insert(
"/sw.js".into(),
AssetFile {
h: hash10(sw.as_bytes()),
s: sw.len(),
g: "core".into(),
},
);
}
if !index.is_empty() {
bytes.insert("/index.html".into(), index.as_bytes().to_vec());
}
if !sw.is_empty() {
bytes.insert("/sw.js".into(), sw.as_bytes().to_vec());
}
delivery_files.extend(files.clone());
let manifest =
json!({"buildTag":build_tag,"files":files,"groups":groups,"contracts":contracts});
let etags = bytes
.iter()
.map(|(path, content)| (path.clone(), etag(content)))
.collect();
let bytes = bytes
.into_iter()
.map(|(path, content)| (path, Bytes::from(content)))
.collect();
Ok(AssetSnapshot {
build_tag,
manifest,
delivery_files,
bytes,
etags,
})
}
pub fn cache_control(
path: &str,
version: Option<&str>,
current: Option<&AssetFile>,
hashing: bool,
build_tag: &str,
) -> &'static str {
if !hashing && path == "/index.html" {
return "no-cache";
}
if let Some(version) = version {
if current.map(|f| f.h.as_str()) == Some(version) || (!hashing && version == build_tag) {
return "public, max-age=31536000, immutable";
}
return if hashing { "no-store" } else { "no-cache" };
}
if path.starts_with("/fonts/") || path.starts_with("/icons/") {
"public, max-age=2592000"
} else {
"no-cache"
}
}
#[cfg(test)]
mod tests {
use super::{cache_control, embed, stamp, AssetFile};
use std::{collections::BTreeMap, path::Path};
#[test]
fn repository_frontend_snapshot_has_hashed_assets_and_injected_tags() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frontend");
let snapshot = super::load(&root, true, true, None).expect("frontend asset manifest");
assert_eq!(snapshot.build_tag.len(), 12);
assert!(
snapshot.manifest["files"]["/app.js"]["h"]
.as_str()
.unwrap()
.len()
== 10
);
assert!(snapshot.bytes["/index.html"]
.windows(snapshot.build_tag.len())
.any(|w| w == snapshot.build_tag.as_bytes()));
let sw = String::from_utf8(snapshot.bytes["/sw.js"].to_vec()).unwrap();
assert!(sw.contains(&snapshot.build_tag));
let index = String::from_utf8(snapshot.bytes["/index.html"].to_vec()).unwrap();
assert!(index.contains(&format!(
"styles.css?v={}",
snapshot.manifest["files"]["/styles.css"]["h"]
.as_str()
.unwrap()
)));
assert!(index.contains(&format!(
r#""/app.js":{{"h":"{}""#,
snapshot.manifest["files"]["/app.js"]["h"].as_str().unwrap()
)));
}
#[test]
fn rollback_snapshot_uses_legacy_single_tag() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frontend");
let snapshot =
super::load(&root, false, false, Some("legacy-build")).expect("rollback snapshot");
assert_eq!(snapshot.build_tag, "legacy-build");
let index = String::from_utf8(snapshot.bytes["/index.html"].to_vec()).unwrap();
assert!(index.contains("styles.css?v=legacy-build"));
assert!(index.contains(r#""/app.js":{"h":"legacy-build""#));
assert!(index.contains(r#""appCache":false"#));
}
#[test]
fn legacy_stamping_only_changes_relative_urls_in_resource_tags() {
let input = r#"<link rel="stylesheet" href="styles.css"><script src="/root.js"></script><script src="https://cdn.test/app.js"></script><a href="other.js">link</a>"#;
let output = stamp(input, &BTreeMap::new(), "legacy-tag", false);
assert!(output.contains(r#"href="styles.css?v=legacy-tag""#));
assert!(output.contains(r#"src="/root.js""#));
assert!(output.contains(r#"src="https://cdn.test/app.js""#));
assert!(output.contains(r#"href="other.js""#));
}
#[test]
fn legacy_index_embeds_single_build_tag_for_each_asset() {
let file = AssetFile {
h: "file-hash".into(),
s: 10,
g: "core".into(),
};
let files = BTreeMap::from([("/app.js".to_string(), file)]);
let groups = serde_json::json!({"core":{"files":["/app.js"]}});
let source = r#"<script type="application/json" id="ytp-assets">{}</script>"#;
let output = embed(source, &files, &groups, "legacy-tag", false, false);
assert!(output.contains(r#""h":"legacy-tag""#));
assert!(!output.contains("file-hash"));
}
#[test]
fn cache_header_matrix_uses_legacy_build_tag_and_file_hashes() {
let file = AssetFile {
h: "abc123".into(),
s: 10,
g: "core".into(),
};
assert_eq!(
cache_control(
"/app.js",
Some("legacy-tag"),
Some(&file),
false,
"legacy-tag"
),
"public, max-age=31536000, immutable"
);
assert_eq!(
cache_control("/app.js", Some("old-tag"), Some(&file), false, "legacy-tag"),
"no-cache"
);
assert_eq!(
cache_control("/app.js", Some("abc123"), Some(&file), true, "build"),
"public, max-age=31536000, immutable"
);
assert_eq!(
cache_control("/app.js", Some("stale"), Some(&file), true, "build"),
"no-store"
);
}
}