From e99e53ff6dc68e27b1a1fc38209980fb1ff3367b Mon Sep 17 00:00:00 2001 From: Felix Roos Date: Wed, 11 Jan 2023 19:56:36 +0100 Subject: [PATCH] experiment: render audio with web-audio-api-rs --- packages/webaudio/render.mjs | 313 ++++++++++++++++++ website/node-playback-test.mjs | 36 ++ website/package-lock.json | 91 ++++- website/package.json | 4 +- website/src/pages/audio/example-[name].wav.js | 52 +++ 5 files changed, 494 insertions(+), 2 deletions(-) create mode 100644 packages/webaudio/render.mjs create mode 100644 website/node-playback-test.mjs create mode 100644 website/src/pages/audio/example-[name].wav.js diff --git a/packages/webaudio/render.mjs b/packages/webaudio/render.mjs new file mode 100644 index 000000000..1a2047a23 --- /dev/null +++ b/packages/webaudio/render.mjs @@ -0,0 +1,313 @@ +import { toMidi, fromMidi } from '@strudel.cycles/core/util.mjs'; +// import './feedbackdelay.mjs'; +// import './reverb.mjs'; + +// TODO: reuse as much logic as possible between webaudio.mjs and render.mjs + +const logger = console.log; + +export const splitSN = (s, n) => { + if (!s.includes(':')) { + return [s, n]; + } + let [s2, n2] = s.split(':'); + if (isNaN(Number(n2))) { + return [s, n]; + } + return [s2, n2]; +}; + +const getOscillator = (ac, { s, freq, t, duration, release }) => { + // make oscillator + const o = ac.createOscillator(); + o.type = s || 'triangle'; + o.frequency.value = Number(freq); + o.start(t); + o.stop(t + duration + release); + return o; +}; +const getADSR = (ac, attack, decay, sustain, release, velocity, begin, end) => { + const gainNode = ac.createGain(); + gainNode.gain.setValueAtTime(0, begin); + gainNode.gain.linearRampToValueAtTime(velocity, begin + attack); // attack + gainNode.gain.linearRampToValueAtTime(sustain * velocity, begin + attack + decay); // sustain start + gainNode.gain.setValueAtTime(sustain * velocity, end); // sustain end + gainNode.gain.linearRampToValueAtTime(0, end + release); // release + // for some reason, using exponential ramping creates little cracklings + /* let t = begin; + gainNode.gain.setValueAtTime(0, t); + gainNode.gain.exponentialRampToValueAtTime(velocity, (t += attack)); + const sustainGain = Math.max(sustain * velocity, 0.001); + gainNode.gain.exponentialRampToValueAtTime(sustainGain, (t += decay)); + if (end - begin < attack + decay) { + gainNode.gain.cancelAndHoldAtTime(end); + } else { + gainNode.gain.setValueAtTime(sustainGain, end); + } + gainNode.gain.exponentialRampToValueAtTime(0.001, end + release); // release */ + return gainNode; +}; + +function gainNode(ac, value) { + const node = ac.createGain(); + node.gain.value = value; + return node; +} + +/* let delays = {}; +function getDelay(ac, orbit, delaytime, delayfeedback, t) { + if (!delays[orbit]) { + const dly = ac.createFeedbackDelay(1, delaytime, delayfeedback); + dly.start?.(t); // for some reason, this throws when audion extension is installed.. + dly.connect(ac.destination); + delays[orbit] = dly; + } + delays[orbit].delayTime.value !== delaytime && delays[orbit].delayTime.setValueAtTime(delaytime, t); + delays[orbit].feedback.value !== delayfeedback && delays[orbit].feedback.setValueAtTime(delayfeedback, t); + return delays[orbit]; +} + +let reverbs = {}; +function getReverb(ac, orbit, duration = 2) { + if (!reverbs[orbit]) { + const reverb = ac.createReverb(duration); + reverb.connect(ac.destination); + reverbs[orbit] = reverb; + } + if (reverbs[orbit].duration !== duration) { + reverbs[orbit] = reverbs[orbit].setDuration(duration); + reverbs[orbit].duration = duration; + } + return reverbs[orbit]; +} */ + +function effectSend(input, effect, wet) { + const send = gainNode(wet); + input.connect(send); + send.connect(effect); + return send; +} + +const getFilter = (ac, type, frequency, Q) => { + const filter = ac.createBiquadFilter(); + filter.type = type; + filter.frequency.value = frequency; + filter.Q.value = Q; + return filter; +}; + +// export const webaudioOutput = async (t, hap, ct, cps) => { +export const renderHap = async (ac, hap, deadline) => { + const hapDuration = hap.duration; + /* if (isNote(hap.value)) { + // supports primitive hap values that look like notes + hap.value = { note: hap.value }; + } */ + if (typeof hap.value !== 'object') { + logger( + `hap.value "${hap.value}" is not supported by webaudio output. Hint: append .note() or .s() to the end`, + 'error', + ); + /* throw new Error( + `hap.value "${hap.value}"" is not supported by webaudio output. Hint: append .note() or .s() to the end`, + ); */ + return; + } + // calculate correct time (tone.js workaround) + let t = ac.currentTime + deadline; + // destructure value + let { + freq, + s, + bank, + sf, + clip = 0, // if 1, samples will be cut off when the hap ends + n = 0, + note, + gain = 0.8, + // low pass + lpf, + cutoff = lpf, + lpq = 1, + resonance = lpq, + // high pass + hpf, + hcutoff = hpf, + hpq = 1, + hresonance = hpq, + // band pass + bpf, + bandf = bpf, + bpq = 1, + bandq = bpq, + // + coarse, + crush, + shape, + pan, + speed = 1, // sample playback speed + begin = 0, + end = 1, + vowel, + delay = 0, + delayfeedback = 0.5, + delaytime = 0.25, + unit, + nudge = 0, // TODO: is this in seconds? + cut, + loop, + orbit = 1, + room, + size = 2, + roomsize = size, + } = hap.value; + const { velocity = 1 } = hap.context; + gain *= velocity; // legacy fix for velocity + // the chain will hold all audio nodes that connect to each other + const chain = []; + if (bank && s) { + s = `${bank}_${s}`; + } + if (typeof s === 'string') { + [s, n] = splitSN(s, n); + } + if (typeof note === 'string') { + [note, n] = splitSN(note, n); + } + if (!s || ['sine', 'square', 'triangle', 'sawtooth'].includes(s)) { + // destructure adsr here, because the default should be different for synths and samples + const { attack = 0.001, decay = 0.05, sustain = 0.6, release = 0.01 } = hap.value; + // with synths, n and note are the same thing + n = note || n || 36; + if (typeof n === 'string') { + n = toMidi(n); // e.g. c3 => 48 + } + // get frequency + if (!freq && typeof n === 'number') { + freq = fromMidi(n); // + 48); + } + // make oscillator + const o = getOscillator(ac, { t, s, freq, duration: hapDuration, release }); + chain.push(o); + // level down oscillators as they are really loud compared to samples i've tested + chain.push(gainNode(ac, 0.3)); + // TODO: make adsr work with samples without pops + // envelope + const adsr = getADSR(ac, attack, decay, sustain, release, 1, t, t + hapDuration); + chain.push(adsr); + } else { + // TODO: refactor sampler to node + /* + // destructure adsr here, because the default should be different for synths and samples + const { attack = 0.001, decay = 0.001, sustain = 1, release = 0.001 } = hap.value; + // load sample + if (speed === 0) { + // no playback + return; + } + if (!s) { + console.warn('no sample specified'); + return; + } + const soundfont = getSoundfontKey(s); + let bufferSource; + + if (soundfont) { + // is soundfont + bufferSource = await globalThis.getFontBufferSource(soundfont, note || n, ac, freq); + } else { + // is sample from loaded samples(..) + bufferSource = await getSampleBufferSource(s, n, note, speed, freq); + } + // asny stuff above took too long? + if (ac.currentTime > t) { + logger(`[sampler] still loading sound "${s}:${n}"`, 'highlight'); + // console.warn('sample still loading:', s, n); + return; + } + if (!bufferSource) { + console.warn('no buffer source'); + return; + } + bufferSource.playbackRate.value = Math.abs(speed) * bufferSource.playbackRate.value; + if (unit === 'c') { + // are there other units? + bufferSource.playbackRate.value = bufferSource.playbackRate.value * bufferSource.buffer.duration; + } + let duration = soundfont || clip ? hapDuration : bufferSource.buffer.duration / bufferSource.playbackRate.value; + // "The computation of the offset into the sound is performed using the sound buffer's natural sample rate, + // rather than the current playback rate, so even if the sound is playing at twice its normal speed, + // the midway point through a 10-second audio buffer is still 5." + const offset = begin * duration * bufferSource.playbackRate.value; + duration = (end - begin) * duration; + if (loop) { + bufferSource.loop = true; + bufferSource.loopStart = offset; + bufferSource.loopEnd = offset + duration; + duration = loop * duration; + } + t += nudge; + + bufferSource.start(t, offset); + if (cut !== undefined) { + cutGroups[cut]?.stop(t); // fade out? + cutGroups[cut] = bufferSource; + } + chain.push(bufferSource); + bufferSource.stop(t + duration + release); + const adsr = getADSR(ac, attack, decay, sustain, release, 1, t, t + duration); + chain.push(adsr); */ + } + + // gain stage + chain.push(gainNode(ac, gain)); + + // filters + cutoff !== undefined && chain.push(getFilter(ac, 'lowpass', cutoff, resonance)); + hcutoff !== undefined && chain.push(getFilter(ac, 'highpass', hcutoff, hresonance)); + bandf !== undefined && chain.push(getFilter(ac, 'bandpass', bandf, bandq)); + vowel !== undefined && chain.push(ac.createVowelFilter(vowel)); + + // effects + /* coarse !== undefined && chain.push(getWorklet(ac, 'coarse-processor', { coarse })); + crush !== undefined && chain.push(getWorklet(ac, 'crush-processor', { crush })); + shape !== undefined && chain.push(getWorklet(ac, 'shape-processor', { shape })); */ + + // panning + if (pan !== undefined) { + const panner = ac.createStereoPanner(); + panner.pan.value = 2 * pan - 1; + chain.push(panner); + } + + // last gain + const post = gainNode(ac, 1); + chain.push(post); + // post.connect(getDestination()); + post.connect(ac.destination); + + // TODO: refactor delay / reverb for node: extending nodes does not work.. + // delay + /* let delaySend; + if (delay > 0 && delaytime > 0 && delayfeedback > 0) { + const delyNode = getDelay(ac, orbit, delaytime, delayfeedback, t); + delaySend = effectSend(post, delyNode, delay); + } + // reverb + let reverbSend; + if (room > 0 && roomsize > 0) { + const reverbNode = getReverb(ac, orbit, roomsize); + reverbSend = effectSend(post, reverbNode, room); + } */ + + // connect chain elements together + chain.slice(1).reduce((last, current) => last.connect(current), chain[0]); + + // disconnect all nodes when source node has ended: + chain[0].onended = () => + chain + .concat([ + /*delaySend , reverbSend */ + ]) + .forEach((n) => n?.disconnect()); +}; diff --git a/website/node-playback-test.mjs b/website/node-playback-test.mjs new file mode 100644 index 000000000..0367a080d --- /dev/null +++ b/website/node-playback-test.mjs @@ -0,0 +1,36 @@ +import { AudioContext } from 'node-web-audio-api'; +import { evaluate } from '@strudel.cycles/transpiler'; +import { renderHap } from '@strudel.cycles/webaudio/render.mjs'; +import { evalScope, controls } from '@strudel.cycles/core'; +import { zeldasRescue as tune } from './src/repl/tunes.mjs'; + +// tunes with cracks and pops: meltingsubmarine, hyperpop, juxUndTollerei (once) +// tunes not working: loungeSponge (csound), undergroundPlumber (draw) + +await evalScope( + controls, // + import('@strudel.cycles/core'), // + import('@strudel.cycles/mini'), // + import('@strudel.cycles/tonal'), // + // import('@strudel.cycles/webaudio'), // currently fails in node: uses document + { + samples: () => { + console.log('samples() skipped'); + }, + loadCsound: () => { + console.log('csound output is not available in node yet..'); + }, + }, +); + +const audioContext = new AudioContext(); +const latency = 0.2; +const seconds = 20; + +const { pattern } = await evaluate(tune); +pattern + .queryArc(0, seconds) + .filter((hap) => hap.hasOnset() && hap.whole.begin >= 0) + .map((hap) => { + renderHap(audioContext, hap, hap.whole.begin + latency); + }); diff --git a/website/package-lock.json b/website/package-lock.json index c3e750105..f835223c5 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -33,7 +33,9 @@ "tailwindcss": "^3.2.4" }, "devDependencies": { - "html-escaper": "^3.0.3" + "audiobuffer-to-wav": "^1.0.0", + "html-escaper": "^3.0.3", + "node-web-audio-api": "^0.5.0" } }, "node_modules/@algolia/autocomplete-core": { @@ -910,6 +912,37 @@ "rollup": ">=2" } }, + "node_modules/@napi-rs/cli": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.14.2.tgz", + "integrity": "sha512-g2bGIhu+p/+wZJlNYGYbb2feQs7vF0dKIMJY8jOBdhU4r14TqxBPEr7P/xoFzHiJ7vCNdJ0rldcC+nOSnJwCTw==", + "dev": true, + "bin": { + "napi": "scripts/index.js" + }, + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/triples": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/triples/-/triples-1.1.0.tgz", + "integrity": "sha512-XQr74QaLeMiqhStEhLn1im9EOMnkypp7MZOwQhGzqp2Weu5eQJbpPxWxixxlYRKWPOmJjsk6qYfYH9kq43yc2w==", + "dev": true + }, + "node_modules/@node-rs/helper": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@node-rs/helper/-/helper-1.3.3.tgz", + "integrity": "sha512-p4OdfQObGN9YFy5WZaGwlPYICQSe7xZYyXB0sxREmvj1HzGKp5bPg2PlfgfMZEfnjIA882B9ZrnagYzZihIwjA==", + "dev": true, + "dependencies": { + "@napi-rs/triples": "^1.1.0" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1594,6 +1627,12 @@ "node": ">=10" } }, + "node_modules/audiobuffer-to-wav": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/audiobuffer-to-wav/-/audiobuffer-to-wav-1.0.0.tgz", + "integrity": "sha512-CAoir4NRrAzAgYo20tEMiKZR84coE8bq/L+H2kwAaULVY4+0xySsEVtNT5raqpzmH6y0pqzY6EmoViLd9W8F/w==", + "dev": true + }, "node_modules/autoprefixer": { "version": "10.4.13", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz", @@ -5167,6 +5206,19 @@ "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.8.tgz", "integrity": "sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A==" }, + "node_modules/node-web-audio-api": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/node-web-audio-api/-/node-web-audio-api-0.5.0.tgz", + "integrity": "sha512-HMFQHti2RnhgAzIrK6PSpDqgwKHMdeWsVILBwEQcCWzol9nGDK+QQQCJPWpMTfvTuGsfEletzRqz+p6fhcjDuA==", + "dev": true, + "dependencies": { + "@napi-rs/cli": "^2.10.0", + "@node-rs/helper": "^1.3.3" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -8171,6 +8223,27 @@ "vfile": "^5.0.0" } }, + "@napi-rs/cli": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.14.2.tgz", + "integrity": "sha512-g2bGIhu+p/+wZJlNYGYbb2feQs7vF0dKIMJY8jOBdhU4r14TqxBPEr7P/xoFzHiJ7vCNdJ0rldcC+nOSnJwCTw==", + "dev": true + }, + "@napi-rs/triples": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/triples/-/triples-1.1.0.tgz", + "integrity": "sha512-XQr74QaLeMiqhStEhLn1im9EOMnkypp7MZOwQhGzqp2Weu5eQJbpPxWxixxlYRKWPOmJjsk6qYfYH9kq43yc2w==", + "dev": true + }, + "@node-rs/helper": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@node-rs/helper/-/helper-1.3.3.tgz", + "integrity": "sha512-p4OdfQObGN9YFy5WZaGwlPYICQSe7xZYyXB0sxREmvj1HzGKp5bPg2PlfgfMZEfnjIA882B9ZrnagYzZihIwjA==", + "dev": true, + "requires": { + "@napi-rs/triples": "^1.1.0" + } + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -8760,6 +8833,12 @@ } } }, + "audiobuffer-to-wav": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/audiobuffer-to-wav/-/audiobuffer-to-wav-1.0.0.tgz", + "integrity": "sha512-CAoir4NRrAzAgYo20tEMiKZR84coE8bq/L+H2kwAaULVY4+0xySsEVtNT5raqpzmH6y0pqzY6EmoViLd9W8F/w==", + "dev": true + }, "autoprefixer": { "version": "10.4.13", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz", @@ -11056,6 +11135,16 @@ "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.8.tgz", "integrity": "sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A==" }, + "node-web-audio-api": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/node-web-audio-api/-/node-web-audio-api-0.5.0.tgz", + "integrity": "sha512-HMFQHti2RnhgAzIrK6PSpDqgwKHMdeWsVILBwEQcCWzol9nGDK+QQQCJPWpMTfvTuGsfEletzRqz+p6fhcjDuA==", + "dev": true, + "requires": { + "@napi-rs/cli": "^2.10.0", + "@node-rs/helper": "^1.3.3" + } + }, "normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", diff --git a/website/package.json b/website/package.json index 2101c41a2..2305e3084 100644 --- a/website/package.json +++ b/website/package.json @@ -37,6 +37,8 @@ "tailwindcss": "^3.2.4" }, "devDependencies": { - "html-escaper": "^3.0.3" + "audiobuffer-to-wav": "^1.0.0", + "html-escaper": "^3.0.3", + "node-web-audio-api": "^0.5.0" } } diff --git a/website/src/pages/audio/example-[name].wav.js b/website/src/pages/audio/example-[name].wav.js new file mode 100644 index 000000000..3b59d5c33 --- /dev/null +++ b/website/src/pages/audio/example-[name].wav.js @@ -0,0 +1,52 @@ +import { OfflineAudioContext } from 'node-web-audio-api'; +import toWav from 'audiobuffer-to-wav'; +import { evaluate } from '@strudel.cycles/transpiler'; +import { renderHap } from '@strudel.cycles/webaudio/render.mjs'; +import { evalScope, controls } from '@strudel.cycles/core'; +import * as tunes from '../../repl/tunes.mjs'; + +await evalScope( + controls, // + import('@strudel.cycles/core'), // + import('@strudel.cycles/mini'), // + import('@strudel.cycles/tonal'), // + // import('@strudel.cycles/webaudio'), // need document.. + { + samples: () => { + console.log('samples() skipped'); + }, + }, +); + +export async function get({ params, request }) { + const { name } = params; + const tune = tunes[name]; + const channels = 1; + const seconds = 20; + const sampleRate = 44100; + const offline = new OfflineAudioContext(channels, sampleRate * seconds, sampleRate); + + // const { pattern } = await evaluate(`"(3,8)".off(1/8, add(7)).note().s('sawtooth').lpf(2000)`); + const { pattern } = await evaluate(tune); + pattern + .queryArc(0, seconds) + .filter((hap) => hap.hasOnset() && hap.whole.begin >= 0) + .map((hap) => { + renderHap(offline, hap, hap.whole.begin); + }); + + const buffer = await offline.startRendering(); + const wav = toWav(buffer); + return { + body: wav, + encoding: 'binary', + }; +} + +export function getStaticPaths() { + return Object.keys(tunes).map((name) => ({ + params: { + name, + }, + })); +}