73 lines
2.5 KiB
JavaScript
73 lines
2.5 KiB
JavaScript
// This module, TensorFlow and the model are imported only after Transcribe.
|
|
let enginePromise, model;
|
|
const MODEL = 'https://cdn.jsdelivr.net/npm/@spotify/basic-pitch@1.0.1/model/model.json';
|
|
async function engine() {
|
|
if (!enginePromise)
|
|
enginePromise = (async () => {
|
|
const tf = await import('https://esm.sh/@tensorflow/tfjs@4.22.0');
|
|
await selectBackend(
|
|
tf,
|
|
!!navigator.gpu,
|
|
() => import('https://esm.sh/@tensorflow/tfjs-backend-webgpu@4.22.0'),
|
|
);
|
|
await tf.ready();
|
|
const pitch = await import(
|
|
'https://esm.sh/@spotify/basic-pitch@1.0.1?deps=@tensorflow/tfjs@4.22.0'
|
|
);
|
|
const graph = await tf.loadGraphModel(MODEL);
|
|
model = new pitch.BasicPitch(Promise.resolve(graph));
|
|
return { tf, pitch, backend: tf.getBackend() };
|
|
})().catch((error) => {
|
|
enginePromise = null;
|
|
throw error;
|
|
});
|
|
return enginePromise;
|
|
}
|
|
export async function selectBackend(tf, hasGPU, loadGPU) {
|
|
if (hasGPU) {
|
|
try {
|
|
await loadGPU();
|
|
if (await tf.setBackend('webgpu')) return 'webgpu';
|
|
} catch {}
|
|
}
|
|
for (const backend of ['webgl', 'cpu']) {
|
|
try {
|
|
if (await tf.setBackend(backend)) return backend;
|
|
} catch {}
|
|
}
|
|
throw new Error('No TensorFlow backend is available in this browser.');
|
|
}
|
|
export async function transcribe(buffer, progress) {
|
|
const decoded = await new OfflineAudioContext(1, 1, 22050).decodeAudioData(buffer);
|
|
if (decoded.duration > 900) throw new Error('Browser transcription is limited to 15 minutes.');
|
|
const context = new OfflineAudioContext(1, Math.ceil(decoded.duration * 22050), 22050),
|
|
source = context.createBufferSource();
|
|
source.buffer = decoded;
|
|
source.connect(context.destination);
|
|
source.start();
|
|
const audio = await context.startRendering();
|
|
progress('Loading Basic Pitch library and model…');
|
|
const { tf, pitch, backend } = await engine();
|
|
const frames = [],
|
|
onsets = [];
|
|
// Basic Pitch 1.0.1 retains intermediate tensors; release each run's scope.
|
|
// Model weights were loaded before this scope and stay available for reuse.
|
|
tf.engine().startScope();
|
|
try {
|
|
await model.evaluateModel(
|
|
audio.getChannelData(0),
|
|
(f, o) => {
|
|
frames.push(...f);
|
|
onsets.push(...o);
|
|
},
|
|
(p) => progress(`Transcribing on ${backend}: ${Math.round(p * 100)}%`),
|
|
);
|
|
} finally {
|
|
tf.engine().endScope();
|
|
}
|
|
return {
|
|
notes: pitch.noteFramesToTime(pitch.outputToNotesPoly(frames, onsets, 0.25, 0.25, 5)),
|
|
engine: `basic-pitch-1.0.1/${backend}`,
|
|
};
|
|
}
|