Add notes backup management
This commit is contained in:
Jonathan Sykes
@@ -646,6 +646,15 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="p-backups" class="panel">
|
||||
<h1 class="page">Backups</h1>
|
||||
<p class="lead">Daily note snapshots can be inspected, downloaded, and restored. Every restore writes a separate safety snapshot first.</p>
|
||||
<div class="card"><div class="row"><h2 style="margin:0">Notes snapshots</h2><span class="grow"></span><button class="btn pri" id="backupNow" type="button">Create backup now</button><a class="btn sm" href="/api/admin/backups/export/sqlite-dump">Export SQLite dump</a></div>
|
||||
<p id="backupStatus" class="msg" role="status" aria-live="polite"></p><div class="scroll"><table class="tbl" id="backupTable"></table></div>
|
||||
</div>
|
||||
<div class="card"><h2>Preview and restore</h2><div class="row"><label>Backup date<select class="field" id="backupDate"></select></label><label>Optional song ID<input class="field" id="backupSong" placeholder="Restore one song only" /></label><button class="btn" id="backupPreview" type="button">Preview diff</button><button class="btn dng" id="backupRestore" type="button">Restore</button></div><div id="backupDiff" class="muted"></div></div>
|
||||
</section>
|
||||
|
||||
<!-- ================= ACCESS ================= -->
|
||||
<section id="p-analytics" class="panel">
|
||||
<h1 class="page">Server analytics</h1>
|
||||
@@ -774,12 +783,13 @@
|
||||
library: '<path d="M9 18V6l10-2v12"/><circle cx="6.5" cy="18" r="2.5"/><circle cx="16.5" cy="16" r="2.5"/>',
|
||||
peers: '<circle cx="6" cy="12" r="2.5"/><circle cx="18" cy="6" r="2.5"/><circle cx="18" cy="18" r="2.5"/><path d="M8.3 10.9l7.4-3.8M8.3 13.1l7.4 3.8"/>',
|
||||
history: '<path d="M3 12a9 9 0 109-9 9 9 0 00-6.4 2.6L3 8"/><path d="M3 3v5h5M12 7v5l3 2"/>',
|
||||
backups: '<path d="M4 5h16v15H4zM8 5V3h8v2M8 11h8M8 15h5"/>',
|
||||
analytics: '<path d="M4 20h16M7 16V9M12 16V4M17 16v-5"/>',
|
||||
'deep-search': '<circle cx="10.5" cy="10.5" r="6.5"/><path d="m15.5 15.5 5 5M8 10.5h5M10.5 8v5"/>',
|
||||
audit: '<path d="M5 4h14v16H5zM8 8h8M8 12h8M8 16h5"/>',
|
||||
access: '<circle cx="8" cy="15" r="4"/><path d="M11 12l9-9M16 7l3 3M14 9l2 2"/>',
|
||||
};
|
||||
const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
|
||||
const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']];
|
||||
let tab = 'editor';
|
||||
function buildTabs() {
|
||||
const html = TABS.map(([k, l]) => `<button class="tab" role="tab" data-tab="${k}" aria-selected="false"><svg viewBox="0 0 24 24">${ICONS[k]}</svg><span>${l}</span>${k === 'reports' ? '<span class="badge hidden" data-badge>0</span>' : ''}</button>`).join('');
|
||||
@@ -798,6 +808,7 @@
|
||||
if (name === 'cache') loadCacheHealth();
|
||||
if (name === 'index') loadIndexHealth();
|
||||
if (name === 'history') loadRevs();
|
||||
if (name === 'backups') loadBackups();
|
||||
if (name === 'peers') loadP2p();
|
||||
if (name === 'library') loadUploads();
|
||||
if (name === 'access') loadTokens();
|
||||
@@ -987,6 +998,43 @@
|
||||
$('auditPrev').addEventListener('click', () => { auditOffset = Math.max(0, auditOffset - 50); loadAudit(); });
|
||||
$('auditNext').addEventListener('click', () => { auditOffset += 50; loadAudit(); });
|
||||
|
||||
/* ---------------- note backups ---------------- */
|
||||
async function loadBackups() {
|
||||
const table = $('backupTable'); table.innerHTML = '<tr><td class="muted">Loading backups…</td></tr>';
|
||||
try {
|
||||
const result = await api('/api/admin/backups'); if (!result.ok) throw Error(result.error || 'Backup list unavailable.');
|
||||
const valid = result.backups.filter(item => item.valid);
|
||||
$('backupDate').innerHTML = valid.map(item => `<option value="${esc(item.date)}">${esc(item.date)} · ${item.notes} notes · ${bytes(item.bytes)}</option>`).join('');
|
||||
table.innerHTML = result.backups.length ? `<tr><th>Date</th><th>Size</th><th>Notes</th><th>Integrity</th><th></th></tr>${result.backups.map(item => `<tr><td>${esc(item.date)}</td><td>${bytes(item.bytes)}</td><td>${item.notes}</td><td>${item.valid ? 'Verified' : 'Invalid JSON'}</td><td>${item.valid ? `<a class="btn sm" href="/api/admin/backups/${encodeURIComponent(item.date)}/download">Download</a><button class="btn sm" data-backup-preview="${esc(item.date)}">Preview</button>` : ''}</td></tr>`).join('')}` : '<tr><td class="muted">No daily backups found yet.</td></tr>';
|
||||
$('backupPreview').disabled = $('backupRestore').disabled = !valid.length;
|
||||
if (valid.length) await previewBackup();
|
||||
} catch (error) { table.innerHTML = `<tr><td class="err">${esc(error.message)}</td></tr>`; }
|
||||
}
|
||||
async function previewBackup() {
|
||||
const date = $('backupDate').value; if (!date) return;
|
||||
const result = await api(`/api/admin/backups/${encodeURIComponent(date)}`);
|
||||
if (!result.ok) { $('backupDiff').textContent = result.error || 'Could not preview backup.'; return; }
|
||||
const diff = result.diff;
|
||||
$('backupDiff').innerHTML = `<p>${esc(date)} · ${diff.changed} changed · ${diff.added} only in backup · ${diff.removed} only live</p><ul>${diff.preview.map(item => `<li><code>${esc(item.key)}</code> — ${esc(item.change)}</li>`).join('') || '<li>No differences</li>'}</ul>`;
|
||||
}
|
||||
$('backupNow').addEventListener('click', async () => {
|
||||
$('backupStatus').textContent = 'Writing snapshot…';
|
||||
const result = await api('/api/admin/backups', { method: 'POST', body: {} });
|
||||
$('backupStatus').className = `msg ${result.ok ? 'ok' : 'err'}`; $('backupStatus').textContent = result.ok ? `Saved ${result.notes} notes to ${result.date}.` : result.error || 'Backup failed.';
|
||||
await loadBackups();
|
||||
});
|
||||
$('backupDate').addEventListener('change', previewBackup); $('backupPreview').addEventListener('click', previewBackup);
|
||||
$('backupTable').addEventListener('click', event => { const button = event.target.closest('[data-backup-preview]'); if (button) { $('backupDate').value = button.dataset.backupPreview; previewBackup(); } });
|
||||
$('backupRestore').addEventListener('click', async () => {
|
||||
const date = $('backupDate').value, videoId = $('backupSong').value.trim(); if (!date) return;
|
||||
const scope = videoId ? `song ${videoId}` : 'all shared notes';
|
||||
if (!confirm(`Restore ${scope} from ${date}? A safety snapshot will be created first.`)) return;
|
||||
$('backupStatus').textContent = 'Creating safety snapshot and restoring…';
|
||||
const result = await api(`/api/admin/backups/${encodeURIComponent(date)}/restore`, { method: 'POST', body: videoId ? { videoId } : {} });
|
||||
$('backupStatus').className = `msg ${result.ok ? 'ok' : 'err'}`; $('backupStatus').textContent = result.ok ? `Restored ${result.restored} note documents. Safety snapshot: ${result.safetySnapshot}` : result.error || 'Restore failed.';
|
||||
await loadBackups();
|
||||
});
|
||||
|
||||
/* ---------------- lyrics quality queue ---------------- */
|
||||
let qualityOffset = 0;
|
||||
async function loadLyricsQuality() {
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
const { defineConfig } = require('@playwright/test');
|
||||
const classic = require('./playwright.classic.config');
|
||||
module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index|reports)\.spec\.js/ });
|
||||
module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index|reports|backups)\.spec\.js/ });
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { audit } from './audit-log.js';
|
||||
import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const clientOf = db => typeof db?.execute === 'function' ? db : db?.db;
|
||||
const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } };
|
||||
@@ -7,6 +9,109 @@ const number = value => Number(value) || 0;
|
||||
export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null, backfillStep = null }) {
|
||||
const sql = clientOf(db);
|
||||
|
||||
const backupFile = date => join(backupDir || '.', `notes-${date}.json`);
|
||||
const backupDate = value => /^\d{4}-\d{2}-\d{2}$/.test(value || '') && !Number.isNaN(Date.parse(`${value}T00:00:00Z`));
|
||||
async function readBackup(date) {
|
||||
if (!backupDate(date)) return null;
|
||||
try {
|
||||
const data = JSON.parse(await readFile(backupFile(date), 'utf8'));
|
||||
if (!Array.isArray(data.notes) || data.notes.some(note => !note?.videoId || !['lyrics', 'chapters'].includes(note.kind) || !note.data)) return null;
|
||||
return data;
|
||||
} catch { return null; }
|
||||
}
|
||||
async function createBackup({ safety = false } = {}) {
|
||||
if (!backupDir || typeof db.allNotes !== 'function') throw new Error('Backup storage unavailable');
|
||||
await mkdir(safety ? join(backupDir, 'snapshots') : backupDir, { recursive: true });
|
||||
const now = new Date();
|
||||
const date = now.toISOString().slice(0, 10);
|
||||
const file = safety ? join(backupDir, 'snapshots', `notes-${date}-${now.toISOString().slice(11, 19).replaceAll(':', '-')}-safety.json`) : backupFile(date);
|
||||
const payload = { exportedAt: now.toISOString(), notes: await db.allNotes() };
|
||||
await writeFile(file, JSON.stringify(payload));
|
||||
return { file, date, notes: payload.notes.length };
|
||||
}
|
||||
|
||||
app.get('/api/admin/backups', adminAuth, async c => {
|
||||
if (!backupDir) return c.json({ ok: false, error: 'Backup storage unavailable' }, 503);
|
||||
let names = []; try { names = await readdir(backupDir); } catch {}
|
||||
const backups = [];
|
||||
for (const name of names.filter(value => /^notes-\d{4}-\d{2}-\d{2}\.json$/.test(value)).sort().reverse()) {
|
||||
const date = name.slice(6, 16); const data = await readBackup(date); const info = await stat(backupFile(date)).catch(() => null);
|
||||
backups.push({ date, bytes: info?.size || 0, exportedAt: data?.exportedAt || null, notes: data?.notes.length || 0, valid: !!data });
|
||||
}
|
||||
return c.json({ ok: true, backups }, 200, { 'Cache-Control': 'no-store' });
|
||||
});
|
||||
|
||||
app.get('/api/admin/backups/:date', adminAuth, async c => {
|
||||
const data = await readBackup(c.req.param('date'));
|
||||
if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404);
|
||||
const live = await db.allNotes();
|
||||
const index = rows => new Map(rows.map(row => [`${row.videoId}:${row.kind}`, JSON.stringify(row.data)]));
|
||||
const old = index(data.notes), current = index(live);
|
||||
const added = [...old.keys()].filter(key => !current.has(key));
|
||||
const removed = [...current.keys()].filter(key => !old.has(key));
|
||||
const changed = [...old.keys()].filter(key => current.has(key) && old.get(key) !== current.get(key));
|
||||
return c.json({ ok: true, date: c.req.param('date'), exportedAt: data.exportedAt, total: data.notes.length,
|
||||
diff: { added: added.length, removed: removed.length, changed: changed.length, preview: [...changed.slice(0, 30).map(key => ({ key, change: 'changed' })), ...added.slice(0, 15).map(key => ({ key, change: 'only in backup' })), ...removed.slice(0, 15).map(key => ({ key, change: 'only live' }))] } });
|
||||
});
|
||||
|
||||
app.get('/api/admin/backups/:date/download', adminAuth, async c => {
|
||||
const data = await readBackup(c.req.param('date'));
|
||||
if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404);
|
||||
c.header('Content-Disposition', `attachment; filename="notes-${c.req.param('date')}.json"`);
|
||||
return c.json(data);
|
||||
});
|
||||
|
||||
app.post('/api/admin/backups', adminAuth, async c => {
|
||||
try {
|
||||
const result = await createBackup();
|
||||
await audit(db, 'admin', 'backup.create', result.date, { notes: result.notes });
|
||||
return c.json({ ok: true, ...result });
|
||||
} catch (error) { return c.json({ ok: false, error: error.message }, 503); }
|
||||
});
|
||||
|
||||
app.post('/api/admin/backups/:date/restore', adminAuth, async c => {
|
||||
const date = c.req.param('date'); const data = await readBackup(date);
|
||||
if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404);
|
||||
let body = {}; try { body = await c.req.json(); } catch {}
|
||||
const selected = body.videoId ? data.notes.filter(note => note.videoId === body.videoId) : data.notes;
|
||||
if (body.videoId && !selected.length) return c.json({ ok: false, error: 'Song is not present in this backup' }, 404);
|
||||
const safety = await createBackup({ safety: true });
|
||||
for (const note of selected) await db.saveNote({ videoId: note.videoId, kind: note.kind, data: note.data, source: 'restore', updatedBy: `admin (backup ${date})`, force: true });
|
||||
if (!body.videoId) {
|
||||
const keep = new Set(data.notes.map(note => `${note.videoId}:${note.kind}`));
|
||||
for (const note of await db.allNotes()) if (!keep.has(`${note.videoId}:${note.kind}`)) {
|
||||
await sql.execute({ sql: 'DELETE FROM video_notes WHERE video_id=? AND kind=?', args: [note.videoId, note.kind] });
|
||||
}
|
||||
}
|
||||
await audit(db, 'admin', 'backup.restore', date, { videoId: body.videoId || null, restored: selected.length, safetySnapshot: safety.file });
|
||||
return c.json({ ok: true, restored: selected.length, safetySnapshot: safety.file });
|
||||
});
|
||||
|
||||
app.get('/api/admin/backups/export/sqlite-dump', adminAuth, async c => {
|
||||
const schema = await sql.execute("SELECT type,name,sql FROM sqlite_master WHERE type IN ('table','index','trigger') AND name NOT LIKE 'sqlite_%' AND sql IS NOT NULL ORDER BY CASE type WHEN 'table' THEN 0 ELSE 1 END,name");
|
||||
const quote = value => {
|
||||
if (value == null) return 'NULL';
|
||||
if (typeof value === 'number') return Number.isFinite(value) ? String(value) : 'NULL';
|
||||
if (typeof value === 'bigint') return String(value);
|
||||
if (value instanceof Uint8Array) return `X'${Buffer.from(value).toString('hex')}'`;
|
||||
return `'${String(value).replaceAll("'", "''")}'`;
|
||||
};
|
||||
const statements = ['PRAGMA foreign_keys=OFF;', 'BEGIN TRANSACTION;'];
|
||||
for (const table of schema.rows.filter(row => row.type === 'table')) {
|
||||
statements.push(`${table.sql};`);
|
||||
const rows = await sql.execute(`SELECT * FROM "${table.name.replaceAll('"', '""')}"`);
|
||||
for (const row of rows.rows) {
|
||||
const columns = Object.keys(row);
|
||||
if (!columns.length) continue;
|
||||
statements.push(`INSERT INTO "${table.name}" (${columns.map(name => `"${name.replaceAll('"', '""')}"`).join(',')}) VALUES (${columns.map(name => quote(row[name])).join(',')});`);
|
||||
}
|
||||
}
|
||||
for (const object of schema.rows.filter(row => row.type !== 'table')) statements.push(`${object.sql};`);
|
||||
statements.push('COMMIT;');
|
||||
c.header('Content-Disposition', 'attachment; filename="ytplayer-notes-dump.sql"');
|
||||
return c.text(statements.join('\n'), 200, { 'Content-Type': 'application/sql; charset=utf-8' });
|
||||
});
|
||||
|
||||
app.get('/api/admin/lyrics/quality', adminAuth, async c => {
|
||||
const minLines = Math.max(1, Math.min(200, Number(c.req.query('minLines')) || 8));
|
||||
const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0));
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { Hono } from 'hono';
|
||||
@@ -10,11 +11,12 @@ process.env.MEDIA_CACHE_PINNED_MAX_BYTES = '1000';
|
||||
const db = await import('./db.js');
|
||||
const { registerAdminToolsRoutes } = await import('./admin-tools.js');
|
||||
const app = new Hono();
|
||||
const backupDir = join(root, 'backups');
|
||||
const auth = async (c, next) => c.req.header('x-admin') === 'yes' ? next() : c.json({ ok: false }, 401);
|
||||
|
||||
beforeAll(async () => {
|
||||
await db.initDb();
|
||||
registerAdminToolsRoutes(app, { db, adminAuth: auth, backfillStep: async () => 3 });
|
||||
registerAdminToolsRoutes(app, { db, adminAuth: auth, backupDir, backfillStep: async () => 3 });
|
||||
await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] });
|
||||
await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] });
|
||||
await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] });
|
||||
@@ -70,4 +72,25 @@ describe('lyrics quality queue', () => {
|
||||
expect(clean.removed).toBeGreaterThanOrEqual(1);
|
||||
expect((await db.db.execute({ sql: "SELECT COUNT(*) AS n FROM video_meta WHERE id='OldOrphan01'" })).rows[0].n).toBe(0);
|
||||
});
|
||||
|
||||
test('backup list verifies, previews, downloads, safely restores and exports a SQLite dump', async () => {
|
||||
const headers = { 'x-admin': 'yes', 'content-type': 'application/json' };
|
||||
await db.saveNote({ videoId: 'BackupSong01', kind: 'lyrics', data: { lines: [{ t: 1, text: 'Before' }] }, force: true, source: 'user' });
|
||||
const created = await (await app.request('/api/admin/backups', { method: 'POST', headers, body: '{}' })).json();
|
||||
expect(created.ok).toBe(true);
|
||||
await db.saveNote({ videoId: 'BackupSong01', kind: 'lyrics', data: { lines: [{ t: 1, text: 'After' }] }, force: true, source: 'user' });
|
||||
const list = await (await app.request('/api/admin/backups', { headers })).json();
|
||||
expect(list.backups.find(item => item.date === created.date)).toMatchObject({ valid: true, notes: created.notes });
|
||||
const preview = await (await app.request(`/api/admin/backups/${created.date}`, { headers })).json();
|
||||
expect(preview.diff.changed).toBe(1);
|
||||
const download = await app.request(`/api/admin/backups/${created.date}/download`, { headers });
|
||||
expect(download.status).toBe(200);
|
||||
const restored = await (await app.request(`/api/admin/backups/${created.date}/restore`, { method: 'POST', headers, body: JSON.stringify({ videoId: 'BackupSong01' }) })).json();
|
||||
expect(restored.ok).toBe(true);
|
||||
expect((await db.getNotes('BackupSong01')).lyrics.data.lines[0].text).toBe('Before');
|
||||
expect(existsSync(restored.safetySnapshot)).toBe(true);
|
||||
const dump = await app.request('/api/admin/backups/export/sqlite-dump', { headers: { 'x-admin': 'yes' } });
|
||||
expect(dump.status).toBe(200);
|
||||
expect(await dump.text()).toContain('CREATE TABLE');
|
||||
});
|
||||
});
|
||||
|
||||
25
tests/admin-backups.spec.js
Normal file
25
tests/admin-backups.spec.js
Normal file
@@ -0,0 +1,25 @@
|
||||
const { test, expect } = require('@playwright/test');
|
||||
|
||||
test('backups lists integrity, previews diff, creates a snapshot and confirms restore', async ({ page }) => {
|
||||
const calls = [];
|
||||
await page.route('**/api/**', async route => {
|
||||
const request = route.request(), url = new URL(request.url()); calls.push({ path: url.pathname, method: request.method() });
|
||||
let body = { ok: true };
|
||||
if (url.pathname === '/api/admin/me') body = { ok: true, enabled: true, admin: true };
|
||||
if (url.pathname === '/api/admin/backups' && request.method() === 'GET') body = { ok: true, backups: [{ date: '2026-10-14', bytes: 1024, notes: 12, valid: true }] };
|
||||
if (url.pathname === '/api/admin/backups/2026-10-14') body = { ok: true, diff: { changed: 2, added: 1, removed: 0, preview: [{ key: 'song:lyrics', change: 'changed' }] } };
|
||||
if (url.pathname === '/api/admin/backups' && request.method() === 'POST') body = { ok: true, date: '2026-10-14', notes: 12 };
|
||||
if (url.pathname.endsWith('/restore')) body = { ok: true, restored: 12, safetySnapshot: '/data/backups/snapshots/safety.json' };
|
||||
await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) });
|
||||
});
|
||||
page.on('dialog', dialog => dialog.accept());
|
||||
await page.goto('/admin.html#backups');
|
||||
await expect(page.locator('#backupTable')).toContainText('Verified');
|
||||
await expect(page.locator('#backupDiff')).toContainText('2 changed');
|
||||
await expect(page.locator('a[href="/api/admin/backups/export/sqlite-dump"]')).toBeVisible();
|
||||
await page.locator('#backupNow').click();
|
||||
await expect(page.locator('#backupStatus')).toContainText('Saved 12 notes');
|
||||
await page.locator('#backupRestore').click();
|
||||
await expect.poll(() => calls.some(call => call.path.endsWith('/restore') && call.method === 'POST')).toBe(true);
|
||||
await expect(page.locator('#backupStatus')).toContainText('Safety snapshot');
|
||||
});
|
||||
Reference in New Issue
Block a user