From 229dfe487aa614743d12f71288f53d43e270bf28 Mon Sep 17 00:00:00 2001 From: Vasilii Milovidov Date: Tue, 26 Sep 2023 18:42:15 +0400 Subject: [PATCH] first go at handling samples from the web --- packages/desktopbridge/webaudiobridge.mjs | 341 +++++++++++----------- src-tauri/src/webaudiobridge.rs | 90 +++--- 2 files changed, 225 insertions(+), 206 deletions(-) diff --git a/packages/desktopbridge/webaudiobridge.mjs b/packages/desktopbridge/webaudiobridge.mjs index 6710524c9..6ab246aee 100644 --- a/packages/desktopbridge/webaudiobridge.mjs +++ b/packages/desktopbridge/webaudiobridge.mjs @@ -1,181 +1,196 @@ -import {Invoke} from './utils.mjs'; -import {duration, isNote, midiToFreq, noteToMidi, Pattern} from '@strudel.cycles/core'; -import {getAudioContext, getEnvelope, getSound} from '@strudel.cycles/webaudio'; +import { Invoke } from './utils.mjs'; +import { duration, isNote, midiToFreq, noteToMidi, Pattern, valueToMidi } from '@strudel.cycles/core'; +import { getAudioContext, getEnvelope, getSound, logger } from '@strudel.cycles/webaudio'; export const desktopAudio = async (value, deadline, hapDuration) => { - const ac = getAudioContext(); - if (typeof value !== 'object') { - throw new Error( - `expected hap.value to be an object, but got "${value}". Hint: append .note() or .s() to the end`, - 'error', - ); + const ac = getAudioContext(); + if (typeof value !== 'object') { + throw new Error( + `expected hap.value to be an object, but got "${value}". Hint: append .note() or .s() to the end`, + 'error', + ); + } + + let t = ac.currentTime + deadline; + + let { + note = 'C2', + s, + bank, + source, + gain = 0.8, + // low pass + cutoff = 8000, + resonance = 1, + // high pass + hcutoff = 0, + hresonance = 1, + // band pass + bandf = 0, + bandq = 1, + // + coarse, + crush, + shape, + pan, + vowel, + delay = 0, + delayfeedback = 0.5, + delaytime = 0.25, + orbit = 1, + room, + size = 2, + velocity = 1, + analyze, // analyser wet + fft = 8, // fftSize 0 - 10 + speed = 1, // sample playback speed + begin = 0, + end = 1, + loop = 0, + loopBegin = 0, + loopEnd = 1, + attack = 0.001, + decay = 0.05, + sustain = 1, + release = 0.1, + lpattack = 0.0001, + lpdecay = 0.2, + lpsustain = 0.6, + lprelease = 0.2, + lpenv = 1, + hpattack = 0.0001, + hpdecay = 0.2, + hpsustain = 0.6, + hprelease = 0.2, + hpenv = 1, + bpattack = 0.0001, + bpdecay = 0.2, + bpsustain = 0.6, + bprelease = 0.2, + bpenv = 1, + n = 0, + freq, + } = value; + + if (bank && s) { + s = `${bank}_${s}`; + } + let map = getSound(s).data.samples; + + if (freq !== undefined && note !== undefined) { + logger('[sampler] hap has note and freq. ignoring note', 'warning'); + } + let midi = valueToMidi({ freq, note }, 36); + value.duration = hapDuration; + let transpose; + transpose = midi - 36; + let sampleUrl; + let baseUrl; + if (s === 'sine' || s === 'square' || s === 'saw' || s === 'triangle') { + sampleUrl = 'none'; + } else { + let path; + if (getSound(s).data.baseUrl !== undefined) { + baseUrl = getSound(s).data.baseUrl; + console.log('baseUrl', baseUrl); + if (baseUrl === './piano/') { + path = 'https://strudel.tidalcycles.org/'; + } else if (baseUrl === './EmuSP12/') { + path = 'https://strudel.tidalcycles.org/'; + } else { + path = ''; + } } - let t = ac.currentTime + deadline; - - let { - note = 'C3', - s, - bank = '', - source, - gain = 0.8, - // low pass - cutoff = 8000, - resonance = 1, - // high pass - hcutoff = 0, - hresonance = 1, - // band pass - bandf = 0, - bandq = 1, - // - coarse, - crush, - shape, - pan, - vowel, - delay = 0, - delayfeedback = 0.5, - delaytime = 0.25, - orbit = 1, - room, - size = 2, - velocity = 1, - analyze, // analyser wet - fft = 8, // fftSize 0 - 10 - speed = 1, // sample playback speed - begin = 0, - end = 1, - loop = 0, - loopBegin = 0, - loopEnd = 1, - attack = 0.001, - decay = 0.05, - sustain = 1, - release = 0.1, - lpattack = 0.0001, - lpdecay = 0.2, - lpsustain = 0.6, - lprelease = 0.2, - lpenv = 1, - hpattack = 0.0001, - hpdecay = 0.2, - hpsustain = 0.6, - hprelease = 0.2, - hpenv = 1, - bpattack = 0.0001, - bpdecay = 0.2, - bpsustain = 0.6, - bprelease = 0.2, - bpenv = 1, - n = 0, - } = value; - - if (isNote(note)) { - note = noteToMidi(note); - } - value.duration = hapDuration; - let transpose = 0; - transpose = note - 36; - let sampleUrl; - if (s === 'sine' || s === 'square' || s === 'saw' || s === 'triangle') { - sampleUrl = 'none'; + if (Array.isArray(map)) { + sampleUrl = + path !== undefined ? path + map[n % map.length].replace('./', '') : map[n % map.length].replace('./', ''); } else { - bank = getSound(s).data.samples; - console.log('bank', bank) - - // if (bank && s) { - // s = `${bank}_${s}`; - // } - - let path = ''; - if (getSound(s).data.baseUrl !== undefined) { - const baseUrl = getSound(s).data.baseUrl; - if (baseUrl === './piano/') { - path = 'https://strudel.tidalcycles.org/'; - } else if (baseUrl === './EmuSP12/') { - path = 'https://strudel.tidalcycles.org/'; - } - } - - if (Array.isArray(bank)) { - sampleUrl = path !== undefined ? path + bank[n % bank.length].replace('./', '') : bank[n % bank.length].replace('./', ''); - } else { - const midiDiff = (noteA) => noteToMidi(noteA) - note; - // object format will expect keys as notes - const closest = Object.keys(bank) - .filter((k) => !k.startsWith('_')) - .reduce( - (closest, key, j) => (!closest || Math.abs(midiDiff(key)) < Math.abs(midiDiff(closest)) ? key : closest), - null, - ); - transpose = -midiDiff(closest); // semitones to repitch - sampleUrl = path !== undefined ? path + bank[closest][n % bank[closest].length].replace('./', '') : bank[closest][n % bank[closest].length].replace('./', ''); - } + const midiDiff = (noteA) => noteToMidi(noteA) - midi; + // object format will expect keys as notes + const closest = Object.keys(map) + .filter((k) => !k.startsWith('_')) + .reduce( + (closest, key, j) => (!closest || Math.abs(midiDiff(key)) < Math.abs(midiDiff(closest)) ? key : closest), + null, + ); + console.log('closest', closest); + transpose = -midiDiff(closest); // semitones to repitch + sampleUrl = + path !== undefined + ? path + map[closest][n % map[closest].length].replace('./', '') + : map[closest][n % map[closest].length].replace('./', ''); } - console.log('sampleUrl', sampleUrl) - const playbackRate = 1.0 * Math.pow(2, transpose / 12); + } + if (isNote(note)) { + note = noteToMidi(note); + } + const playbackRate = 1.0 * Math.pow(2, transpose / 12); - if (delay !== 0) { - delay = Math.abs(delay); - delayfeedback = Math.abs(delayfeedback); - delaytime = Math.abs(delaytime); - } + if (delay !== 0) { + delay = Math.abs(delay); + delayfeedback = Math.abs(delayfeedback); + delaytime = Math.abs(delaytime); + } - let adsr_on = attack !== 0.001 || decay !== 0.05 || sustain !== 1 || release !== 0.01 ? 1 : 0; + let adsr_on = attack !== 0.001 || decay !== 0.05 || sustain !== 1 || release !== 0.01 ? 1 : 0; - const packages = { - loop: [loop, loopBegin, loopEnd], - delay: [delay, delaytime, delayfeedback], - lpf: [cutoff, resonance], - hpf: [hcutoff, hresonance], - bpf: [bandf, bandq], - adsr: [attack, decay, sustain, release, adsr_on], - lpenv: [lpattack, lpdecay, lpsustain, lprelease, lpenv], - hpenv: [hpattack, hpdecay, hpsustain, hprelease, hpenv], - bpenv: [bpattack, bpdecay, bpsustain, bprelease, bpenv], - }; - const offset = (t - getAudioContext().currentTime) * 1000; - const roundedOffset = Math.round(offset); - const messagesfromjs = []; - messagesfromjs.push({ - note: midiToFreq(note), - offset: roundedOffset, - waveform: s, - lpf: packages.lpf, - hpf: packages.hpf, - bpf: packages.bpf, - duration: hapDuration, - velocity: velocity, - delay: packages.delay, - orbit: orbit, - speed: speed * playbackRate, - begin: begin, - end: end, - looper: packages.loop, - adsr: packages.adsr, - lpenv: packages.lpenv, - hpenv: packages.hpenv, - bpenv: packages.bpenv, - n: n, - sampleurl: sampleUrl, + const packages = { + loop: [loop, loopBegin, loopEnd], + delay: [delay, delaytime, delayfeedback], + lpf: [cutoff, resonance], + hpf: [hcutoff, hresonance], + bpf: [bandf, bandq], + adsr: [attack, decay, sustain, release, adsr_on], + lpenv: [lpattack, lpdecay, lpsustain, lprelease, lpenv], + hpenv: [hpattack, hpdecay, hpsustain, hprelease, hpenv], + bpenv: [bpattack, bpdecay, bpsustain, bprelease, bpenv], + }; + + const dirname = bank + '/' + s + '/'; + + const offset = (t - getAudioContext().currentTime) * 1000; + const roundedOffset = Math.round(offset); + const messagesfromjs = []; + messagesfromjs.push({ + note: midiToFreq(note), + offset: roundedOffset, + waveform: s, + lpf: packages.lpf, + hpf: packages.hpf, + bpf: packages.bpf, + duration: hapDuration, + velocity: velocity, + delay: packages.delay, + orbit: orbit, + speed: speed * playbackRate, + begin: begin, + end: end, + looper: packages.loop, + adsr: packages.adsr, + lpenv: packages.lpenv, + hpenv: packages.hpenv, + bpenv: packages.bpenv, + n: n, + sampleurl: sampleUrl, + dirname: dirname, + }); + + if (messagesfromjs.length) { + setTimeout(() => { + Invoke('sendwebaudio', { messagesfromjs }); }); - - if (messagesfromjs.length) { - setTimeout(() => { - Invoke('sendwebaudio', {messagesfromjs}); - }); - } + } }; const hap2value = (hap) => { - hap.ensureObjectValue(); - return {...hap.value, velocity: hap.context.velocity}; + hap.ensureObjectValue(); + return { ...hap.value, velocity: hap.context.velocity }; }; export const webaudioDesktopOutputTrigger = (t, hap, ct, cps) => - desktopAudio(hap2value(hap), t - ct, hap.duration / cps, cps); + desktopAudio(hap2value(hap), t - ct, hap.duration / cps, cps); export const webaudioDesktopOutput = (hap, deadline, hapDuration) => - desktopAudio(hap2value(hap), deadline, hapDuration); + desktopAudio(hap2value(hap), deadline, hapDuration); Pattern.prototype.webaudio = function () { - return this.onTrigger(webaudioDesktopOutputTrigger); + return this.onTrigger(webaudioDesktopOutputTrigger); }; diff --git a/src-tauri/src/webaudiobridge.rs b/src-tauri/src/webaudiobridge.rs index 168b004af..2f651e452 100644 --- a/src-tauri/src/webaudiobridge.rs +++ b/src-tauri/src/webaudiobridge.rs @@ -1,22 +1,18 @@ use std::{ sync::Arc, time::Duration, - thread::sleep, }; use std::fs::File; +use std::path::Path; use mini_moka::sync::Cache; use reqwest::Url; -use tokio::{ - sync::{mpsc, Mutex}, - time::Instant, -}; +use tokio::{fs, sync::{mpsc, Mutex}, time::Instant}; use serde::Deserialize; -// use tokio::fs::File; use tokio::io::AsyncWriteExt; -use web_audio_api::{AudioBuffer, context::{AudioContext, AudioContextLatencyCategory, AudioContextOptions}, node::AudioNode}; +use web_audio_api::{AudioBuffer, context::{AudioContext, AudioContextLatencyCategory, AudioContextOptions}}; use web_audio_api::context::BaseAudioContext; -use crate::superdough::{ADSR, BPF, Delay, FilterADSR, HPF, Loop, LPF, sample, superdough, superdough_synth}; +use crate::superdough::{ADSR, BPF, Delay, FilterADSR, HPF, Loop, LPF, superdough_sample, superdough_synth}; #[derive(Debug)] @@ -42,6 +38,7 @@ pub struct WebAudioMessage { pub bpenv: FilterADSR, pub n: usize, pub sampleurl: String, + pub dirname: String, } pub struct AsyncInputTransmitWebAudio { @@ -73,7 +70,9 @@ pub fn init( let message_queue_clone = Arc::clone(&message_queue); - // Create audio context + /* ........................................................... + Prepare audio context + ............................................................*/ let latency_hint = match std::env::var("WEB_AUDIO_LATENCY").as_deref() { Ok("playback") => AudioContextLatencyCategory::Playback, _ => AudioContextLatencyCategory::default(), @@ -82,13 +81,9 @@ pub fn init( latency_hint, ..AudioContextOptions::default() }); - // Create audio context tauri::async_runtime::spawn(async move { - /* ........................................................... - Prepare audio context - ............................................................*/ - let cache:Cache = Cache::new(10_000); + let cache: Cache = Cache::new(10_000); /* ........................................................... Process queued messages ............................................................*/ @@ -103,43 +98,42 @@ pub fn init( match message.waveform.as_str() { "sine" | "square" | "triangle" | "saw" => { - // superdough(message.clone(), &mut audio_context); superdough_synth(message.clone(), &mut audio_context); } _ => { - let url = Url::parse(&*message.sampleurl).unwrap(); - let fname = url.path_segments().and_then(Iterator::last).and_then(|name| if name.is_empty() { None } else { Some(name) }).unwrap_or("tmp.ben"); - let file_path = "/Users/vasiliymilovidov/samples/".to_owned() + fname; - let url_copy = url.clone(); - let file_path_copy = file_path.clone(); + let url = Url::parse(&*message.sampleurl).expect("failed to parse url"); + let filename = url.path_segments() + .and_then(Iterator::last) + .and_then(|name| if name.is_empty() { None } else { Some(name) }) + .unwrap_or("tmp.ben"); + let file_path = format!("samples/{}{}", message.dirname, filename); - match cache.get(&file_path_copy) { - Some(buff) => { - sample(message.clone(), &mut audio_context, buff.clone()); - return false; - } - None => { - match File::open(file_path_copy.clone()) { - Ok(file) => { - let buff = audio_context.decode_audio_data_sync(file).unwrap(); - cache.insert(file_path_copy.clone(), buff.clone()); - } - Err(_) => {} - } - } + if let Some(audio_buffer) = cache.get(&file_path) { + superdough_sample(message.clone(), &mut audio_context, audio_buffer.clone()); + } else if let Ok(file) = File::open(&file_path) { + let audio_buffer = audio_context.decode_audio_data_sync(file) + .unwrap_or_else(|_| panic!("Failed to decode audio data")); + cache.insert(file_path.clone(), audio_buffer.clone()); } + tokio::spawn(async move { - match tokio::fs::metadata(&file_path).await { - Ok(_) => {} - Err(_) => { - let resp = reqwest::get(url_copy).await.unwrap(); - let bytes = resp.bytes().await.unwrap(); - let mut out = tokio::fs::File::create(file_path).await.unwrap(); - out.write_all(&bytes).await.unwrap(); - } + if tokio::fs::metadata(&file_path).await.is_err() { + let response = reqwest::get(url) + .await + .unwrap_or_else(|_| panic!("Failed to send GET request")); + + let bytes = response.bytes().await.unwrap(); + let path = Path::new(&file_path); + let mut file = create_file_and_dirs(path).await; + // let mut file = tokio::fs::File::create(&file_path) + // .await + // .unwrap_or_else(|_| panic!("Failed to create file")); + + file.write_all(&bytes) + .await + .unwrap_or_else(|_| panic!("Failed to write to file")); } }); - } } return false; @@ -182,6 +176,7 @@ pub struct MessageFromJS { bpenv: (f64, f64, f64, f64, f64), n: usize, sampleurl: String, + dirname: String, } // Called from JS @@ -257,9 +252,18 @@ pub async fn sendwebaudio( }, n: m.n, sampleurl: m.sampleurl, + dirname: m.dirname, }; messages_to_process.push(message_to_process); } async_proc_input_tx.send(messages_to_process).await.map_err(|e| e.to_string()) +} + +async fn create_file_and_dirs(path: &Path) -> fs::File { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).await.unwrap(); + } + let file = fs::File::create(path).await.unwrap(); + file } \ No newline at end of file