348 lines
13 KiB
JavaScript
348 lines
13 KiB
JavaScript
/* Opt-in transcription, cached sequences, synced keyboard and practice queue. */
|
||
(function (root) {
|
||
'use strict';
|
||
let adapter,
|
||
database,
|
||
busy = false;
|
||
const memory = new Map();
|
||
async function db() {
|
||
if (!database)
|
||
database = new Promise((resolve, reject) => {
|
||
const req = root.indexedDB.open('ytp-piano', 1);
|
||
req.onupgradeneeded = () => req.result.createObjectStore('notes');
|
||
req.onsuccess = () => resolve(req.result);
|
||
req.onerror = () => reject(req.error);
|
||
});
|
||
return database;
|
||
}
|
||
async function cache(id, value) {
|
||
try {
|
||
const store = (await db())
|
||
.transaction('notes', value ? 'readwrite' : 'readonly')
|
||
.objectStore('notes');
|
||
if (!value)
|
||
return await new Promise((resolve, reject) => {
|
||
const req = store.get(id);
|
||
req.onsuccess = () => resolve(req.result || null);
|
||
req.onerror = () => reject(req.error);
|
||
});
|
||
await new Promise((resolve, reject) => {
|
||
const transaction = store.transaction;
|
||
transaction.oncomplete = resolve;
|
||
transaction.onerror = () => reject(transaction.error);
|
||
store.put(value, id);
|
||
});
|
||
return value;
|
||
} catch {
|
||
if (value) memory.set(id, value);
|
||
return memory.get(id) || null;
|
||
}
|
||
}
|
||
async function audio(id, file) {
|
||
if (file) {
|
||
if (file.size > 100 * 1048576) throw new Error('Choose a file smaller than 100 MB.');
|
||
return file.arrayBuffer();
|
||
}
|
||
const local = await root.OPFS?.getFileObject(id);
|
||
if (local) {
|
||
if (local.size > 100 * 1048576) throw new Error('Use a smaller audio file (100 MB maximum).');
|
||
return local.arrayBuffer();
|
||
}
|
||
const response = await fetch(
|
||
id.startsWith('upl_')
|
||
? `/api/uploads/${encodeURIComponent(id)}`
|
||
: `/api/media/${encodeURIComponent(id)}?a=1`,
|
||
);
|
||
if (!response.ok)
|
||
throw new Error('Save the audio on this device/server, or choose an audio file.');
|
||
const reader = response.body.getReader(),
|
||
chunks = [];
|
||
let size = 0;
|
||
try {
|
||
for (;;) {
|
||
const { value, done } = await reader.read();
|
||
if (done) break;
|
||
size += value.length;
|
||
if (size > 100 * 1048576) throw new Error('Audio exceeds 100 MB.');
|
||
chunks.push(value);
|
||
}
|
||
} finally {
|
||
await reader.cancel().catch(() => {});
|
||
}
|
||
const out = new Uint8Array(size);
|
||
let offset = 0;
|
||
for (const chunk of chunks) {
|
||
out.set(chunk, offset);
|
||
offset += chunk.length;
|
||
}
|
||
return out.buffer;
|
||
}
|
||
function keyboard(canvas, notes, time) {
|
||
const ctx = canvas.getContext('2d'),
|
||
ratio = root.devicePixelRatio || 1,
|
||
width = canvas.clientWidth || 390;
|
||
if (canvas.width !== Math.round(width * ratio)) {
|
||
canvas.width = Math.round(width * ratio);
|
||
canvas.height = 150 * ratio;
|
||
}
|
||
ctx.setTransform(ratio, 0, 0, ratio, 0, 0);
|
||
ctx.clearRect(0, 0, width, 150);
|
||
const active = new Map(root.PianoCore.activeNotes(notes, time).map((n) => [n.pitch, n.hand])),
|
||
black = (p) => [1, 3, 6, 8, 10].includes(p % 12),
|
||
whites = [];
|
||
for (let p = 21; p <= 108; p++) if (!black(p)) whites.push(p);
|
||
const unit = width / whites.length;
|
||
whites.forEach((p, i) => {
|
||
ctx.fillStyle = active.has(p)
|
||
? active.get(p) === 'left'
|
||
? '#62c9e8'
|
||
: '#ffb65c'
|
||
: '#f4f4f2';
|
||
ctx.fillRect(i * unit, 0, unit - 1, 145);
|
||
});
|
||
for (let p = 22; p < 108; p++)
|
||
if (black(p)) {
|
||
const left = whites.filter((n) => n < p).length * unit;
|
||
ctx.fillStyle = active.has(p)
|
||
? active.get(p) === 'left'
|
||
? '#62c9e8'
|
||
: '#ffb65c'
|
||
: '#141619';
|
||
ctx.fillRect(left - unit * 0.32, 0, unit * 0.64, 90);
|
||
}
|
||
}
|
||
function configure(options) {
|
||
adapter = options;
|
||
root.SettingsSections.register({
|
||
id: 'piano',
|
||
title: 'Piano transcription',
|
||
cluster: 'Live & worship',
|
||
summary: () =>
|
||
adapter.settings().pianoEnabled
|
||
? 'On · browser or optional worker'
|
||
: 'Off · load model on request',
|
||
render(container) {
|
||
const doc = container.ownerDocument,
|
||
toggle = doc.createElement('input');
|
||
toggle.type = 'checkbox';
|
||
toggle.checked = !!adapter.settings().pianoEnabled;
|
||
const label = doc.createElement('label');
|
||
label.className = 'set-row';
|
||
label.textContent = 'Enable piano transcription';
|
||
label.append(toggle);
|
||
const note = doc.createElement('p');
|
||
note.textContent =
|
||
'Basic Pitch runs on this device; its library/model download only when you transcribe. Prefers WebGPU, then WebGL/CPU. Best for isolated instruments. The optional server worker must be enabled by an administrator.';
|
||
const file = doc.createElement('input');
|
||
file.type = 'file';
|
||
file.accept = 'audio/*,video/*';
|
||
file.setAttribute('aria-label', 'Optional audio file for the playing song');
|
||
const token = doc.createElement('input');
|
||
token.type = 'password';
|
||
token.autocomplete = 'off';
|
||
token.placeholder = 'Admin API token (server requests only; not saved)';
|
||
token.setAttribute('aria-label', 'Admin API token for piano worker');
|
||
const status = doc.createElement('p');
|
||
status.setAttribute('role', 'status');
|
||
const actions = doc.createElement('div');
|
||
actions.className = 'piano-actions';
|
||
const canvas = doc.createElement('canvas');
|
||
canvas.className = 'piano-keyboard';
|
||
canvas.setAttribute('role', 'img');
|
||
canvas.setAttribute(
|
||
'aria-label',
|
||
'88-key piano: left-hand notes blue, right-hand notes amber',
|
||
);
|
||
const practice = doc.createElement('div');
|
||
practice.className = 'piano-practice';
|
||
container.append(label, note, file, token, actions, status, canvas, practice);
|
||
let sequence = [],
|
||
currentId = '',
|
||
lastTime = -1,
|
||
alive = true;
|
||
const buttons = [];
|
||
const snapshot = () => adapter.state();
|
||
function enabled() {
|
||
for (const b of buttons) b.disabled = busy || !toggle.checked;
|
||
}
|
||
toggle.onchange = () => {
|
||
adapter.settings().pianoEnabled = toggle.checked;
|
||
adapter.persist();
|
||
enabled();
|
||
};
|
||
function button(text, action) {
|
||
const b = doc.createElement('button');
|
||
b.type = 'button';
|
||
b.textContent = text;
|
||
b.onclick = action;
|
||
actions.append(b);
|
||
buttons.push(b);
|
||
return b;
|
||
}
|
||
async function run(action) {
|
||
const id = snapshot().id;
|
||
if (!id) {
|
||
status.textContent = 'Play a song first.';
|
||
return;
|
||
}
|
||
if (busy) return;
|
||
busy = true;
|
||
enabled();
|
||
try {
|
||
await action(id);
|
||
} catch (error) {
|
||
if (alive) status.textContent = error.message;
|
||
} finally {
|
||
busy = false;
|
||
enabled();
|
||
}
|
||
}
|
||
button('Transcribe in browser', () =>
|
||
run(async (id) => {
|
||
const buffer = await audio(id, file.files[0]);
|
||
const engine = await import(root.Lazy?.url('/piano-engine.mjs') || './piano-engine.mjs');
|
||
const result = await engine.transcribe(buffer, (text) => {
|
||
if (alive) status.textContent = text;
|
||
});
|
||
const notes = root.PianoCore.fromBasicPitch(result.notes);
|
||
await cache(id, { notes, engine: result.engine, at: Date.now() });
|
||
if (snapshot().id === id) {
|
||
sequence = notes;
|
||
status.textContent = `${notes.length} notes cached on this device · ${result.engine}. Hand split is a middle-C heuristic.`;
|
||
}
|
||
}),
|
||
);
|
||
button('Request server transcription', () =>
|
||
run(async (id) => {
|
||
const response = await fetch(`/api/media/${encodeURIComponent(id)}/piano`, {
|
||
method: 'POST',
|
||
headers: { Authorization: `Bearer ${token.value}` },
|
||
});
|
||
const result = await response.json();
|
||
if (!response.ok) throw new Error(result.error || 'Server request failed');
|
||
status.textContent = `Server job ${result.job.status}. Use “Load server result” when ready.`;
|
||
}),
|
||
);
|
||
button('Load server result', () =>
|
||
run(async (id) => {
|
||
const response = await fetch(`/api/media/${encodeURIComponent(id)}/piano`),
|
||
result = await response.json();
|
||
if (!response.ok) throw new Error(result.error || 'Server unavailable');
|
||
if (!result.job?.notes) {
|
||
status.textContent = result.enabled
|
||
? `Server job: ${result.job?.status || 'not requested'}`
|
||
: 'The optional server worker is off.';
|
||
return;
|
||
}
|
||
const notes = root.PianoCore.normalizeNotes(result.job.notes);
|
||
await cache(id, { notes, engine: 'bytedance-server', at: Date.now() });
|
||
if (snapshot().id === id) {
|
||
sequence = notes;
|
||
status.textContent = `${notes.length} server notes cached on this device.`;
|
||
}
|
||
}),
|
||
);
|
||
button('Export note JSON', () => {
|
||
if (!sequence.length) {
|
||
status.textContent = 'No note sequence yet.';
|
||
return;
|
||
}
|
||
const url = URL.createObjectURL(
|
||
new Blob([JSON.stringify(sequence)], { type: 'application/json' }),
|
||
);
|
||
const a = doc.createElement('a');
|
||
a.href = url;
|
||
a.download = `${snapshot().id}.notes.json`;
|
||
doc.body.append(a);
|
||
a.click();
|
||
a.remove();
|
||
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
||
});
|
||
function ranges() {
|
||
return adapter.settings().pianoPractice?.[snapshot().id] || [];
|
||
}
|
||
function saveRanges(value) {
|
||
const settings = adapter.settings();
|
||
settings.pianoPractice ||= {};
|
||
settings.pianoPractice[snapshot().id] = value;
|
||
adapter.persist();
|
||
renderPractice();
|
||
}
|
||
function renderPractice() {
|
||
practice.replaceChildren();
|
||
const h = doc.createElement('h4');
|
||
h.textContent = 'Practice queue';
|
||
practice.append(h);
|
||
ranges().forEach((range, i) => {
|
||
const row = doc.createElement('div');
|
||
row.className = 'piano-actions';
|
||
const play = doc.createElement('button');
|
||
play.type = 'button';
|
||
play.textContent = `Loop ${range.start.toFixed(1)}–${range.end.toFixed(1)} s`;
|
||
play.onclick = () => adapter.loop(range.start, range.end);
|
||
const remove = doc.createElement('button');
|
||
remove.type = 'button';
|
||
remove.textContent = 'Remove';
|
||
remove.setAttribute('aria-label', `Remove practice range ${i + 1}`);
|
||
remove.onclick = () => saveRanges(ranges().filter((_, k) => k !== i));
|
||
row.append(play, remove);
|
||
practice.append(row);
|
||
});
|
||
}
|
||
button('Add current 8 s to practice', () => {
|
||
if (!snapshot().id) return;
|
||
const range = root.PianoCore.practiceRange(snapshot().time, snapshot().duration);
|
||
if (range && ranges().length < 50) saveRanges([...ranges(), range]);
|
||
});
|
||
button('Stop practice loop', () => adapter.clearLoop());
|
||
enabled();
|
||
let frame, idleTimer;
|
||
async function tick() {
|
||
if (!alive) return;
|
||
if (!container.isConnected) {
|
||
alive = false;
|
||
token.value = '';
|
||
return;
|
||
}
|
||
if (container.hidden) {
|
||
idleTimer = setTimeout(() => {
|
||
frame = root.requestAnimationFrame(tick);
|
||
}, 250);
|
||
return;
|
||
}
|
||
enabled();
|
||
const state = snapshot();
|
||
if (state.id !== currentId) {
|
||
currentId = state.id;
|
||
sequence = [];
|
||
renderPractice();
|
||
const cached = state.id ? await cache(state.id) : null;
|
||
if (!alive) return;
|
||
if (currentId === state.id) {
|
||
sequence = cached?.notes || [];
|
||
status.textContent = cached
|
||
? `${sequence.length} cached notes · ${cached.engine}`
|
||
: 'Choose a playing song, then transcribe.';
|
||
}
|
||
}
|
||
if (state.time !== lastTime || state.paused) {
|
||
keyboard(canvas, sequence, state.time);
|
||
lastTime = state.time;
|
||
}
|
||
frame = root.requestAnimationFrame(tick);
|
||
}
|
||
frame = root.requestAnimationFrame(tick);
|
||
return () => {
|
||
alive = false;
|
||
clearTimeout(idleTimer);
|
||
root.cancelAnimationFrame(frame);
|
||
token.value = '';
|
||
};
|
||
},
|
||
});
|
||
}
|
||
const api = { configure };
|
||
if (typeof module !== 'undefined') module.exports = api;
|
||
else root.Piano = api;
|
||
})(typeof window !== 'undefined' ? window : globalThis);
|