Add lazy piano transcription and optional leased server worker
This commit is contained in:
Jonathan Sykes
347
frontend/piano.js
Normal file
347
frontend/piano.js
Normal file
@@ -0,0 +1,347 @@
|
||||
/* Opt-in transcription, cached sequences, synced keyboard and practice queue. */
|
||||
(function (root) {
|
||||
'use strict';
|
||||
let adapter,
|
||||
database,
|
||||
busy = false;
|
||||
const memory = new Map();
|
||||
async function db() {
|
||||
if (!database)
|
||||
database = new Promise((resolve, reject) => {
|
||||
const req = root.indexedDB.open('ytp-piano', 1);
|
||||
req.onupgradeneeded = () => req.result.createObjectStore('notes');
|
||||
req.onsuccess = () => resolve(req.result);
|
||||
req.onerror = () => reject(req.error);
|
||||
});
|
||||
return database;
|
||||
}
|
||||
async function cache(id, value) {
|
||||
try {
|
||||
const store = (await db())
|
||||
.transaction('notes', value ? 'readwrite' : 'readonly')
|
||||
.objectStore('notes');
|
||||
if (!value)
|
||||
return await new Promise((resolve, reject) => {
|
||||
const req = store.get(id);
|
||||
req.onsuccess = () => resolve(req.result || null);
|
||||
req.onerror = () => reject(req.error);
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
const transaction = store.transaction;
|
||||
transaction.oncomplete = resolve;
|
||||
transaction.onerror = () => reject(transaction.error);
|
||||
store.put(value, id);
|
||||
});
|
||||
return value;
|
||||
} catch {
|
||||
if (value) memory.set(id, value);
|
||||
return memory.get(id) || null;
|
||||
}
|
||||
}
|
||||
async function audio(id, file) {
|
||||
if (file) {
|
||||
if (file.size > 100 * 1048576) throw new Error('Choose a file smaller than 100 MB.');
|
||||
return file.arrayBuffer();
|
||||
}
|
||||
const local = await root.OPFS?.getFileObject(id);
|
||||
if (local) {
|
||||
if (local.size > 100 * 1048576) throw new Error('Use a smaller audio file (100 MB maximum).');
|
||||
return local.arrayBuffer();
|
||||
}
|
||||
const response = await fetch(
|
||||
id.startsWith('upl_')
|
||||
? `/api/uploads/${encodeURIComponent(id)}`
|
||||
: `/api/media/${encodeURIComponent(id)}?a=1`,
|
||||
);
|
||||
if (!response.ok)
|
||||
throw new Error('Save the audio on this device/server, or choose an audio file.');
|
||||
const reader = response.body.getReader(),
|
||||
chunks = [];
|
||||
let size = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
size += value.length;
|
||||
if (size > 100 * 1048576) throw new Error('Audio exceeds 100 MB.');
|
||||
chunks.push(value);
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel().catch(() => {});
|
||||
}
|
||||
const out = new Uint8Array(size);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
out.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
return out.buffer;
|
||||
}
|
||||
function keyboard(canvas, notes, time) {
|
||||
const ctx = canvas.getContext('2d'),
|
||||
ratio = root.devicePixelRatio || 1,
|
||||
width = canvas.clientWidth || 390;
|
||||
if (canvas.width !== Math.round(width * ratio)) {
|
||||
canvas.width = Math.round(width * ratio);
|
||||
canvas.height = 150 * ratio;
|
||||
}
|
||||
ctx.setTransform(ratio, 0, 0, ratio, 0, 0);
|
||||
ctx.clearRect(0, 0, width, 150);
|
||||
const active = new Map(root.PianoCore.activeNotes(notes, time).map((n) => [n.pitch, n.hand])),
|
||||
black = (p) => [1, 3, 6, 8, 10].includes(p % 12),
|
||||
whites = [];
|
||||
for (let p = 21; p <= 108; p++) if (!black(p)) whites.push(p);
|
||||
const unit = width / whites.length;
|
||||
whites.forEach((p, i) => {
|
||||
ctx.fillStyle = active.has(p)
|
||||
? active.get(p) === 'left'
|
||||
? '#62c9e8'
|
||||
: '#ffb65c'
|
||||
: '#f4f4f2';
|
||||
ctx.fillRect(i * unit, 0, unit - 1, 145);
|
||||
});
|
||||
for (let p = 22; p < 108; p++)
|
||||
if (black(p)) {
|
||||
const left = whites.filter((n) => n < p).length * unit;
|
||||
ctx.fillStyle = active.has(p)
|
||||
? active.get(p) === 'left'
|
||||
? '#62c9e8'
|
||||
: '#ffb65c'
|
||||
: '#141619';
|
||||
ctx.fillRect(left - unit * 0.32, 0, unit * 0.64, 90);
|
||||
}
|
||||
}
|
||||
function configure(options) {
|
||||
adapter = options;
|
||||
root.SettingsSections.register({
|
||||
id: 'piano',
|
||||
title: 'Piano transcription',
|
||||
cluster: 'Live & worship',
|
||||
summary: () =>
|
||||
adapter.settings().pianoEnabled
|
||||
? 'On · browser or optional worker'
|
||||
: 'Off · load model on request',
|
||||
render(container) {
|
||||
const doc = container.ownerDocument,
|
||||
toggle = doc.createElement('input');
|
||||
toggle.type = 'checkbox';
|
||||
toggle.checked = !!adapter.settings().pianoEnabled;
|
||||
const label = doc.createElement('label');
|
||||
label.className = 'set-row';
|
||||
label.textContent = 'Enable piano transcription';
|
||||
label.append(toggle);
|
||||
const note = doc.createElement('p');
|
||||
note.textContent =
|
||||
'Basic Pitch runs on this device; its library/model download only when you transcribe. Prefers WebGPU, then WebGL/CPU. Best for isolated instruments. The optional server worker must be enabled by an administrator.';
|
||||
const file = doc.createElement('input');
|
||||
file.type = 'file';
|
||||
file.accept = 'audio/*,video/*';
|
||||
file.setAttribute('aria-label', 'Optional audio file for the playing song');
|
||||
const token = doc.createElement('input');
|
||||
token.type = 'password';
|
||||
token.autocomplete = 'off';
|
||||
token.placeholder = 'Admin API token (server requests only; not saved)';
|
||||
token.setAttribute('aria-label', 'Admin API token for piano worker');
|
||||
const status = doc.createElement('p');
|
||||
status.setAttribute('role', 'status');
|
||||
const actions = doc.createElement('div');
|
||||
actions.className = 'piano-actions';
|
||||
const canvas = doc.createElement('canvas');
|
||||
canvas.className = 'piano-keyboard';
|
||||
canvas.setAttribute('role', 'img');
|
||||
canvas.setAttribute(
|
||||
'aria-label',
|
||||
'88-key piano: left-hand notes blue, right-hand notes amber',
|
||||
);
|
||||
const practice = doc.createElement('div');
|
||||
practice.className = 'piano-practice';
|
||||
container.append(label, note, file, token, actions, status, canvas, practice);
|
||||
let sequence = [],
|
||||
currentId = '',
|
||||
lastTime = -1,
|
||||
alive = true;
|
||||
const buttons = [];
|
||||
const snapshot = () => adapter.state();
|
||||
function enabled() {
|
||||
for (const b of buttons) b.disabled = busy || !toggle.checked;
|
||||
}
|
||||
toggle.onchange = () => {
|
||||
adapter.settings().pianoEnabled = toggle.checked;
|
||||
adapter.persist();
|
||||
enabled();
|
||||
};
|
||||
function button(text, action) {
|
||||
const b = doc.createElement('button');
|
||||
b.type = 'button';
|
||||
b.textContent = text;
|
||||
b.onclick = action;
|
||||
actions.append(b);
|
||||
buttons.push(b);
|
||||
return b;
|
||||
}
|
||||
async function run(action) {
|
||||
const id = snapshot().id;
|
||||
if (!id) {
|
||||
status.textContent = 'Play a song first.';
|
||||
return;
|
||||
}
|
||||
if (busy) return;
|
||||
busy = true;
|
||||
enabled();
|
||||
try {
|
||||
await action(id);
|
||||
} catch (error) {
|
||||
if (alive) status.textContent = error.message;
|
||||
} finally {
|
||||
busy = false;
|
||||
enabled();
|
||||
}
|
||||
}
|
||||
button('Transcribe in browser', () =>
|
||||
run(async (id) => {
|
||||
const buffer = await audio(id, file.files[0]);
|
||||
const engine = await import('./piano-engine.mjs');
|
||||
const result = await engine.transcribe(buffer, (text) => {
|
||||
if (alive) status.textContent = text;
|
||||
});
|
||||
const notes = root.PianoCore.fromBasicPitch(result.notes);
|
||||
await cache(id, { notes, engine: result.engine, at: Date.now() });
|
||||
if (snapshot().id === id) {
|
||||
sequence = notes;
|
||||
status.textContent = `${notes.length} notes cached on this device · ${result.engine}. Hand split is a middle-C heuristic.`;
|
||||
}
|
||||
}),
|
||||
);
|
||||
button('Request server transcription', () =>
|
||||
run(async (id) => {
|
||||
const response = await fetch(`/api/media/${encodeURIComponent(id)}/piano`, {
|
||||
method: 'POST',
|
||||
headers: { Authorization: `Bearer ${token.value}` },
|
||||
});
|
||||
const result = await response.json();
|
||||
if (!response.ok) throw new Error(result.error || 'Server request failed');
|
||||
status.textContent = `Server job ${result.job.status}. Use “Load server result” when ready.`;
|
||||
}),
|
||||
);
|
||||
button('Load server result', () =>
|
||||
run(async (id) => {
|
||||
const response = await fetch(`/api/media/${encodeURIComponent(id)}/piano`),
|
||||
result = await response.json();
|
||||
if (!response.ok) throw new Error(result.error || 'Server unavailable');
|
||||
if (!result.job?.notes) {
|
||||
status.textContent = result.enabled
|
||||
? `Server job: ${result.job?.status || 'not requested'}`
|
||||
: 'The optional server worker is off.';
|
||||
return;
|
||||
}
|
||||
const notes = root.PianoCore.normalizeNotes(result.job.notes);
|
||||
await cache(id, { notes, engine: 'bytedance-server', at: Date.now() });
|
||||
if (snapshot().id === id) {
|
||||
sequence = notes;
|
||||
status.textContent = `${notes.length} server notes cached on this device.`;
|
||||
}
|
||||
}),
|
||||
);
|
||||
button('Export note JSON', () => {
|
||||
if (!sequence.length) {
|
||||
status.textContent = 'No note sequence yet.';
|
||||
return;
|
||||
}
|
||||
const url = URL.createObjectURL(
|
||||
new Blob([JSON.stringify(sequence)], { type: 'application/json' }),
|
||||
);
|
||||
const a = doc.createElement('a');
|
||||
a.href = url;
|
||||
a.download = `${snapshot().id}.notes.json`;
|
||||
doc.body.append(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
||||
});
|
||||
function ranges() {
|
||||
return adapter.settings().pianoPractice?.[snapshot().id] || [];
|
||||
}
|
||||
function saveRanges(value) {
|
||||
const settings = adapter.settings();
|
||||
settings.pianoPractice ||= {};
|
||||
settings.pianoPractice[snapshot().id] = value;
|
||||
adapter.persist();
|
||||
renderPractice();
|
||||
}
|
||||
function renderPractice() {
|
||||
practice.replaceChildren();
|
||||
const h = doc.createElement('h4');
|
||||
h.textContent = 'Practice queue';
|
||||
practice.append(h);
|
||||
ranges().forEach((range, i) => {
|
||||
const row = doc.createElement('div');
|
||||
row.className = 'piano-actions';
|
||||
const play = doc.createElement('button');
|
||||
play.type = 'button';
|
||||
play.textContent = `Loop ${range.start.toFixed(1)}–${range.end.toFixed(1)} s`;
|
||||
play.onclick = () => adapter.loop(range.start, range.end);
|
||||
const remove = doc.createElement('button');
|
||||
remove.type = 'button';
|
||||
remove.textContent = 'Remove';
|
||||
remove.setAttribute('aria-label', `Remove practice range ${i + 1}`);
|
||||
remove.onclick = () => saveRanges(ranges().filter((_, k) => k !== i));
|
||||
row.append(play, remove);
|
||||
practice.append(row);
|
||||
});
|
||||
}
|
||||
button('Add current 8 s to practice', () => {
|
||||
if (!snapshot().id) return;
|
||||
const range = root.PianoCore.practiceRange(snapshot().time, snapshot().duration);
|
||||
if (range && ranges().length < 50) saveRanges([...ranges(), range]);
|
||||
});
|
||||
button('Stop practice loop', () => adapter.clearLoop());
|
||||
enabled();
|
||||
let frame, idleTimer;
|
||||
async function tick() {
|
||||
if (!alive) return;
|
||||
if (!container.isConnected) {
|
||||
alive = false;
|
||||
token.value = '';
|
||||
return;
|
||||
}
|
||||
if (container.hidden) {
|
||||
idleTimer = setTimeout(() => {
|
||||
frame = root.requestAnimationFrame(tick);
|
||||
}, 250);
|
||||
return;
|
||||
}
|
||||
enabled();
|
||||
const state = snapshot();
|
||||
if (state.id !== currentId) {
|
||||
currentId = state.id;
|
||||
sequence = [];
|
||||
renderPractice();
|
||||
const cached = state.id ? await cache(state.id) : null;
|
||||
if (!alive) return;
|
||||
if (currentId === state.id) {
|
||||
sequence = cached?.notes || [];
|
||||
status.textContent = cached
|
||||
? `${sequence.length} cached notes · ${cached.engine}`
|
||||
: 'Choose a playing song, then transcribe.';
|
||||
}
|
||||
}
|
||||
if (state.time !== lastTime || state.paused) {
|
||||
keyboard(canvas, sequence, state.time);
|
||||
lastTime = state.time;
|
||||
}
|
||||
frame = root.requestAnimationFrame(tick);
|
||||
}
|
||||
frame = root.requestAnimationFrame(tick);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearTimeout(idleTimer);
|
||||
root.cancelAnimationFrame(frame);
|
||||
token.value = '';
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
const api = { configure };
|
||||
if (typeof module !== 'undefined') module.exports = api;
|
||||
else root.Piano = api;
|
||||
})(typeof window !== 'undefined' ? window : globalThis);
|
||||
Reference in New Issue
Block a user