From e3821621f39a7345d0b18b40e7f3c066b3b99359 Mon Sep 17 00:00:00 2001 From: Jonathan Sykes Date: Wed, 14 Oct 2026 02:06:33 +0800 Subject: [PATCH] Add notes backup management --- frontend/admin.html | 50 ++++++++++++++- playwright.admin-audit.config.js | 2 +- server/admin-tools.js | 105 +++++++++++++++++++++++++++++++ server/admin-tools.test.js | 25 +++++++- tests/admin-backups.spec.js | 25 ++++++++ 5 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 tests/admin-backups.spec.js diff --git a/frontend/admin.html b/frontend/admin.html index 268dd62..11a1634 100644 --- a/frontend/admin.html +++ b/frontend/admin.html @@ -646,6 +646,15 @@ +
+

Backups

+

Daily note snapshots can be inspected, downloaded, and restored. Every restore writes a separate safety snapshot first.

+

Notes snapshots

Export SQLite dump
+

+
+

Preview and restore

+
+

Server analytics

@@ -774,12 +783,13 @@ library: '', peers: '', history: '', + backups: '', analytics: '', 'deep-search': '', audit: '', access: '', }; - const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; + const TABS = [['editor', 'Editor'], ['quality', 'Lyrics queue'], ['cache', 'Cache'], ['index', 'Search index'], ['reports', 'Reports'], ['library', 'Library'], ['peers', 'P2P'], ['history', 'History'], ['backups', 'Backups'], ['analytics', 'Stats'], ['deep-search', 'Deep search'], ['audit', 'Audit log'], ['access', 'API']]; let tab = 'editor'; function buildTabs() { const html = TABS.map(([k, l]) => ``).join(''); @@ -798,6 +808,7 @@ if (name === 'cache') loadCacheHealth(); if (name === 'index') loadIndexHealth(); if (name === 'history') loadRevs(); + if (name === 'backups') loadBackups(); if (name === 'peers') loadP2p(); if (name === 'library') loadUploads(); if (name === 'access') loadTokens(); @@ -987,6 +998,43 @@ $('auditPrev').addEventListener('click', () => { auditOffset = Math.max(0, auditOffset - 50); loadAudit(); }); $('auditNext').addEventListener('click', () => { auditOffset += 50; loadAudit(); }); + /* ---------------- note backups ---------------- */ + async function loadBackups() { + const table = $('backupTable'); table.innerHTML = 'Loading backups…'; + try { + const result = await api('/api/admin/backups'); if (!result.ok) throw Error(result.error || 'Backup list unavailable.'); + const valid = result.backups.filter(item => item.valid); + $('backupDate').innerHTML = valid.map(item => ``).join(''); + table.innerHTML = result.backups.length ? `DateSizeNotesIntegrity${result.backups.map(item => `${esc(item.date)}${bytes(item.bytes)}${item.notes}${item.valid ? 'Verified' : 'Invalid JSON'}${item.valid ? `Download` : ''}`).join('')}` : 'No daily backups found yet.'; + $('backupPreview').disabled = $('backupRestore').disabled = !valid.length; + if (valid.length) await previewBackup(); + } catch (error) { table.innerHTML = `${esc(error.message)}`; } + } + async function previewBackup() { + const date = $('backupDate').value; if (!date) return; + const result = await api(`/api/admin/backups/${encodeURIComponent(date)}`); + if (!result.ok) { $('backupDiff').textContent = result.error || 'Could not preview backup.'; return; } + const diff = result.diff; + $('backupDiff').innerHTML = `

${esc(date)} · ${diff.changed} changed · ${diff.added} only in backup · ${diff.removed} only live

    ${diff.preview.map(item => `
  • ${esc(item.key)} — ${esc(item.change)}
    • `).join('') || '
  • No differences
    • '}
`; + } + $('backupNow').addEventListener('click', async () => { + $('backupStatus').textContent = 'Writing snapshot…'; + const result = await api('/api/admin/backups', { method: 'POST', body: {} }); + $('backupStatus').className = `msg ${result.ok ? 'ok' : 'err'}`; $('backupStatus').textContent = result.ok ? `Saved ${result.notes} notes to ${result.date}.` : result.error || 'Backup failed.'; + await loadBackups(); + }); + $('backupDate').addEventListener('change', previewBackup); $('backupPreview').addEventListener('click', previewBackup); + $('backupTable').addEventListener('click', event => { const button = event.target.closest('[data-backup-preview]'); if (button) { $('backupDate').value = button.dataset.backupPreview; previewBackup(); } }); + $('backupRestore').addEventListener('click', async () => { + const date = $('backupDate').value, videoId = $('backupSong').value.trim(); if (!date) return; + const scope = videoId ? `song ${videoId}` : 'all shared notes'; + if (!confirm(`Restore ${scope} from ${date}? A safety snapshot will be created first.`)) return; + $('backupStatus').textContent = 'Creating safety snapshot and restoring…'; + const result = await api(`/api/admin/backups/${encodeURIComponent(date)}/restore`, { method: 'POST', body: videoId ? { videoId } : {} }); + $('backupStatus').className = `msg ${result.ok ? 'ok' : 'err'}`; $('backupStatus').textContent = result.ok ? `Restored ${result.restored} note documents. Safety snapshot: ${result.safetySnapshot}` : result.error || 'Restore failed.'; + await loadBackups(); + }); + /* ---------------- lyrics quality queue ---------------- */ let qualityOffset = 0; async function loadLyricsQuality() { diff --git a/playwright.admin-audit.config.js b/playwright.admin-audit.config.js index a537d91..bcdccd7 100644 --- a/playwright.admin-audit.config.js +++ b/playwright.admin-audit.config.js @@ -1,3 +1,3 @@ const { defineConfig } = require('@playwright/test'); const classic = require('./playwright.classic.config'); -module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index|reports)\.spec\.js/ }); +module.exports = defineConfig({ ...classic, testMatch: /admin-(audit|quality|cache|index|reports|backups)\.spec\.js/ }); diff --git a/server/admin-tools.js b/server/admin-tools.js index fae0f23..d8116a8 100644 --- a/server/admin-tools.js +++ b/server/admin-tools.js @@ -1,4 +1,6 @@ import { audit } from './audit-log.js'; +import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; const clientOf = db => typeof db?.execute === 'function' ? db : db?.db; const parse = value => { try { return JSON.parse(value || '{}'); } catch { return {}; } }; @@ -7,6 +9,109 @@ const number = value => Number(value) || 0; export function registerAdminToolsRoutes(app, { db, adminAuth, media = null, backupDir = null, backfillStep = null }) { const sql = clientOf(db); + const backupFile = date => join(backupDir || '.', `notes-${date}.json`); + const backupDate = value => /^\d{4}-\d{2}-\d{2}$/.test(value || '') && !Number.isNaN(Date.parse(`${value}T00:00:00Z`)); + async function readBackup(date) { + if (!backupDate(date)) return null; + try { + const data = JSON.parse(await readFile(backupFile(date), 'utf8')); + if (!Array.isArray(data.notes) || data.notes.some(note => !note?.videoId || !['lyrics', 'chapters'].includes(note.kind) || !note.data)) return null; + return data; + } catch { return null; } + } + async function createBackup({ safety = false } = {}) { + if (!backupDir || typeof db.allNotes !== 'function') throw new Error('Backup storage unavailable'); + await mkdir(safety ? join(backupDir, 'snapshots') : backupDir, { recursive: true }); + const now = new Date(); + const date = now.toISOString().slice(0, 10); + const file = safety ? join(backupDir, 'snapshots', `notes-${date}-${now.toISOString().slice(11, 19).replaceAll(':', '-')}-safety.json`) : backupFile(date); + const payload = { exportedAt: now.toISOString(), notes: await db.allNotes() }; + await writeFile(file, JSON.stringify(payload)); + return { file, date, notes: payload.notes.length }; + } + + app.get('/api/admin/backups', adminAuth, async c => { + if (!backupDir) return c.json({ ok: false, error: 'Backup storage unavailable' }, 503); + let names = []; try { names = await readdir(backupDir); } catch {} + const backups = []; + for (const name of names.filter(value => /^notes-\d{4}-\d{2}-\d{2}\.json$/.test(value)).sort().reverse()) { + const date = name.slice(6, 16); const data = await readBackup(date); const info = await stat(backupFile(date)).catch(() => null); + backups.push({ date, bytes: info?.size || 0, exportedAt: data?.exportedAt || null, notes: data?.notes.length || 0, valid: !!data }); + } + return c.json({ ok: true, backups }, 200, { 'Cache-Control': 'no-store' }); + }); + + app.get('/api/admin/backups/:date', adminAuth, async c => { + const data = await readBackup(c.req.param('date')); + if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404); + const live = await db.allNotes(); + const index = rows => new Map(rows.map(row => [`${row.videoId}:${row.kind}`, JSON.stringify(row.data)])); + const old = index(data.notes), current = index(live); + const added = [...old.keys()].filter(key => !current.has(key)); + const removed = [...current.keys()].filter(key => !old.has(key)); + const changed = [...old.keys()].filter(key => current.has(key) && old.get(key) !== current.get(key)); + return c.json({ ok: true, date: c.req.param('date'), exportedAt: data.exportedAt, total: data.notes.length, + diff: { added: added.length, removed: removed.length, changed: changed.length, preview: [...changed.slice(0, 30).map(key => ({ key, change: 'changed' })), ...added.slice(0, 15).map(key => ({ key, change: 'only in backup' })), ...removed.slice(0, 15).map(key => ({ key, change: 'only live' }))] } }); + }); + + app.get('/api/admin/backups/:date/download', adminAuth, async c => { + const data = await readBackup(c.req.param('date')); + if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404); + c.header('Content-Disposition', `attachment; filename="notes-${c.req.param('date')}.json"`); + return c.json(data); + }); + + app.post('/api/admin/backups', adminAuth, async c => { + try { + const result = await createBackup(); + await audit(db, 'admin', 'backup.create', result.date, { notes: result.notes }); + return c.json({ ok: true, ...result }); + } catch (error) { return c.json({ ok: false, error: error.message }, 503); } + }); + + app.post('/api/admin/backups/:date/restore', adminAuth, async c => { + const date = c.req.param('date'); const data = await readBackup(date); + if (!data) return c.json({ ok: false, error: 'Backup is missing or invalid' }, 404); + let body = {}; try { body = await c.req.json(); } catch {} + const selected = body.videoId ? data.notes.filter(note => note.videoId === body.videoId) : data.notes; + if (body.videoId && !selected.length) return c.json({ ok: false, error: 'Song is not present in this backup' }, 404); + const safety = await createBackup({ safety: true }); + for (const note of selected) await db.saveNote({ videoId: note.videoId, kind: note.kind, data: note.data, source: 'restore', updatedBy: `admin (backup ${date})`, force: true }); + if (!body.videoId) { + const keep = new Set(data.notes.map(note => `${note.videoId}:${note.kind}`)); + for (const note of await db.allNotes()) if (!keep.has(`${note.videoId}:${note.kind}`)) { + await sql.execute({ sql: 'DELETE FROM video_notes WHERE video_id=? AND kind=?', args: [note.videoId, note.kind] }); + } + } + await audit(db, 'admin', 'backup.restore', date, { videoId: body.videoId || null, restored: selected.length, safetySnapshot: safety.file }); + return c.json({ ok: true, restored: selected.length, safetySnapshot: safety.file }); + }); + + app.get('/api/admin/backups/export/sqlite-dump', adminAuth, async c => { + const schema = await sql.execute("SELECT type,name,sql FROM sqlite_master WHERE type IN ('table','index','trigger') AND name NOT LIKE 'sqlite_%' AND sql IS NOT NULL ORDER BY CASE type WHEN 'table' THEN 0 ELSE 1 END,name"); + const quote = value => { + if (value == null) return 'NULL'; + if (typeof value === 'number') return Number.isFinite(value) ? String(value) : 'NULL'; + if (typeof value === 'bigint') return String(value); + if (value instanceof Uint8Array) return `X'${Buffer.from(value).toString('hex')}'`; + return `'${String(value).replaceAll("'", "''")}'`; + }; + const statements = ['PRAGMA foreign_keys=OFF;', 'BEGIN TRANSACTION;']; + for (const table of schema.rows.filter(row => row.type === 'table')) { + statements.push(`${table.sql};`); + const rows = await sql.execute(`SELECT * FROM "${table.name.replaceAll('"', '""')}"`); + for (const row of rows.rows) { + const columns = Object.keys(row); + if (!columns.length) continue; + statements.push(`INSERT INTO "${table.name}" (${columns.map(name => `"${name.replaceAll('"', '""')}"`).join(',')}) VALUES (${columns.map(name => quote(row[name])).join(',')});`); + } + } + for (const object of schema.rows.filter(row => row.type !== 'table')) statements.push(`${object.sql};`); + statements.push('COMMIT;'); + c.header('Content-Disposition', 'attachment; filename="ytplayer-notes-dump.sql"'); + return c.text(statements.join('\n'), 200, { 'Content-Type': 'application/sql; charset=utf-8' }); + }); + app.get('/api/admin/lyrics/quality', adminAuth, async c => { const minLines = Math.max(1, Math.min(200, Number(c.req.query('minLines')) || 8)); const offset = Math.max(0, Math.min(1_000_000, Number(c.req.query('offset')) || 0)); diff --git a/server/admin-tools.test.js b/server/admin-tools.test.js index d85d21f..6643d32 100644 --- a/server/admin-tools.test.js +++ b/server/admin-tools.test.js @@ -1,5 +1,6 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { mkdtempSync, rmSync } from 'node:fs'; +import { existsSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Hono } from 'hono'; @@ -10,11 +11,12 @@ process.env.MEDIA_CACHE_PINNED_MAX_BYTES = '1000'; const db = await import('./db.js'); const { registerAdminToolsRoutes } = await import('./admin-tools.js'); const app = new Hono(); +const backupDir = join(root, 'backups'); const auth = async (c, next) => c.req.header('x-admin') === 'yes' ? next() : c.json({ ok: false }, 401); beforeAll(async () => { await db.initDb(); - registerAdminToolsRoutes(app, { db, adminAuth: auth, backfillStep: async () => 3 }); + registerAdminToolsRoutes(app, { db, adminAuth: auth, backupDir, backfillStep: async () => 3 }); await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['QualMiss001', JSON.stringify({ title: 'Missing lyric sample', channel: 'Choir' }), 'missing lyric sample choir', 1, Date.now()] }); await db.db.execute({ sql: 'INSERT INTO video_meta(id,card,hay,seen,updated_at) VALUES(?,?,?,?,?)', args: ['Untimely001', JSON.stringify({ title: 'Untimed sample', channel: 'Band' }), 'untimed sample band', 1, Date.now()] }); await db.db.execute({ sql: "INSERT INTO video_notes(video_id,kind,data,rev,source) VALUES(?, 'lyrics', ?, 1, 'user')", args: ['Untimely001', JSON.stringify({ lines: [{ t: null, text: 'A line' }, { t: 4, text: 'Next' }] })] }); @@ -70,4 +72,25 @@ describe('lyrics quality queue', () => { expect(clean.removed).toBeGreaterThanOrEqual(1); expect((await db.db.execute({ sql: "SELECT COUNT(*) AS n FROM video_meta WHERE id='OldOrphan01'" })).rows[0].n).toBe(0); }); + + test('backup list verifies, previews, downloads, safely restores and exports a SQLite dump', async () => { + const headers = { 'x-admin': 'yes', 'content-type': 'application/json' }; + await db.saveNote({ videoId: 'BackupSong01', kind: 'lyrics', data: { lines: [{ t: 1, text: 'Before' }] }, force: true, source: 'user' }); + const created = await (await app.request('/api/admin/backups', { method: 'POST', headers, body: '{}' })).json(); + expect(created.ok).toBe(true); + await db.saveNote({ videoId: 'BackupSong01', kind: 'lyrics', data: { lines: [{ t: 1, text: 'After' }] }, force: true, source: 'user' }); + const list = await (await app.request('/api/admin/backups', { headers })).json(); + expect(list.backups.find(item => item.date === created.date)).toMatchObject({ valid: true, notes: created.notes }); + const preview = await (await app.request(`/api/admin/backups/${created.date}`, { headers })).json(); + expect(preview.diff.changed).toBe(1); + const download = await app.request(`/api/admin/backups/${created.date}/download`, { headers }); + expect(download.status).toBe(200); + const restored = await (await app.request(`/api/admin/backups/${created.date}/restore`, { method: 'POST', headers, body: JSON.stringify({ videoId: 'BackupSong01' }) })).json(); + expect(restored.ok).toBe(true); + expect((await db.getNotes('BackupSong01')).lyrics.data.lines[0].text).toBe('Before'); + expect(existsSync(restored.safetySnapshot)).toBe(true); + const dump = await app.request('/api/admin/backups/export/sqlite-dump', { headers: { 'x-admin': 'yes' } }); + expect(dump.status).toBe(200); + expect(await dump.text()).toContain('CREATE TABLE'); + }); }); diff --git a/tests/admin-backups.spec.js b/tests/admin-backups.spec.js new file mode 100644 index 0000000..79aed34 --- /dev/null +++ b/tests/admin-backups.spec.js @@ -0,0 +1,25 @@ +const { test, expect } = require('@playwright/test'); + +test('backups lists integrity, previews diff, creates a snapshot and confirms restore', async ({ page }) => { + const calls = []; + await page.route('**/api/**', async route => { + const request = route.request(), url = new URL(request.url()); calls.push({ path: url.pathname, method: request.method() }); + let body = { ok: true }; + if (url.pathname === '/api/admin/me') body = { ok: true, enabled: true, admin: true }; + if (url.pathname === '/api/admin/backups' && request.method() === 'GET') body = { ok: true, backups: [{ date: '2026-10-14', bytes: 1024, notes: 12, valid: true }] }; + if (url.pathname === '/api/admin/backups/2026-10-14') body = { ok: true, diff: { changed: 2, added: 1, removed: 0, preview: [{ key: 'song:lyrics', change: 'changed' }] } }; + if (url.pathname === '/api/admin/backups' && request.method() === 'POST') body = { ok: true, date: '2026-10-14', notes: 12 }; + if (url.pathname.endsWith('/restore')) body = { ok: true, restored: 12, safetySnapshot: '/data/backups/snapshots/safety.json' }; + await route.fulfill({ contentType: 'application/json', body: JSON.stringify(body) }); + }); + page.on('dialog', dialog => dialog.accept()); + await page.goto('/admin.html#backups'); + await expect(page.locator('#backupTable')).toContainText('Verified'); + await expect(page.locator('#backupDiff')).toContainText('2 changed'); + await expect(page.locator('a[href="/api/admin/backups/export/sqlite-dump"]')).toBeVisible(); + await page.locator('#backupNow').click(); + await expect(page.locator('#backupStatus')).toContainText('Saved 12 notes'); + await page.locator('#backupRestore').click(); + await expect.poll(() => calls.some(call => call.path.endsWith('/restore') && call.method === 'POST')).toBe(true); + await expect(page.locator('#backupStatus')).toContainText('Safety snapshot'); +});