rust webaudio for tauri tests

packages/desktopbridge/index.mjs

@@ -6,3 +6,4 @@ This program is free software: you can redistribute it and/or modify it under th
 
 export * from './midibridge.mjs';
 export * from './utils.mjs';
+export * from './webaudiobridge.mjs'

packages/desktopbridge/package.json

@@ -21,6 +21,7 @@
       "url": "https://github.com/tidalcycles/strudel/issues"
     },
     "dependencies": {
+        "@strudel.cycles/webaudio": "^0.8.2",
         "@strudel.cycles/core": "workspace:*",
         "@tauri-apps/api": "^1.4.0"
       },

packages/desktopbridge/webaudiobridge.mjs

@@ -0,0 +1,82 @@
+import { Invoke } from './utils.mjs';
+import {isNote, midiToFreq, noteToMidi, Pattern} from '@strudel.cycles/core';
+import {getAudioContext} from "@strudel.cycles/webaudio";
+
+export const desktopAudio = async (value, deadline, hapDuration) => {
+    const ac = getAudioContext();
+    if (typeof value !== 'object') {
+        throw new Error(
+            `expected hap.value to be an object, but got "${value}". Hint: append .note() or .s() to the end`,
+            'error',
+        );
+    }
+
+    let t = ac.currentTime + deadline;
+
+    let {
+        note,
+        s = 'triangle',
+        bank,
+        source,
+        gain = 0.8,
+        // low pass
+        cutoff,
+        resonance = 1,
+        // high pass
+        hcutoff,
+        hresonance = 1,
+        // band pass
+        bandf,
+        bandq = 1,
+        //
+        coarse,
+        crush,
+        shape,
+        pan,
+        vowel,
+        delay = 0,
+        delayfeedback = 0.5,
+        delaytime = 0.25,
+        orbit = 1,
+        room,
+        size = 2,
+        velocity = 1,
+        analyze, // analyser wet
+        fft = 8, // fftSize 0 - 10
+    } = value;
+
+    if (isNote(note)) {
+        note = noteToMidi(note);
+    }
+    const offset = (t - getAudioContext().currentTime) * 1000;
+    const roundedOffset = Math.round(offset);
+    const messagesfromjs = [];
+
+    messagesfromjs.push({
+        note: midiToFreq(note),
+        offset: roundedOffset,
+        waveform: s,
+    });
+
+    if (messagesfromjs.length) {
+        setTimeout(() => {
+            Invoke('sendwebaudio', { messagesfromjs });
+        });
+    }
+}
+const hap2value = (hap) => {
+    hap.ensureObjectValue();
+    return { ...hap.value, velocity: hap.context.velocity };
+};
+export const webaudioDesktopOutputTrigger = (t, hap, ct, cps) => desktopAudio(hap2value(hap), t - ct, hap.duration / cps, cps);
+export const webaudioDesktopOutput = (hap, deadline, hapDuration) => desktopAudio(hap2value(hap), deadline, hapDuration);
+
+Pattern.prototype.webaudio = function () {
+    return this.onTrigger(webaudioDesktopOutputTrigger);
+};
+
+
+
+
+
+

src-tauri/Cargo.toml

@@ -20,6 +20,7 @@ serde = { version = "1.0", features = ["derive"] }
 tauri = { version = "1.4.0", features = ["fs-all"] }
 midir = "0.9.1"
 tokio = { version = "1.29.0", features = ["full"] }
+web-audio-api = "0.33.0"
 
 [features]
 # this feature is used for production builds or when `devPath` points to the filesystem and the built-in dev server is disabled.

src-tauri/src/main.rs

@@ -2,22 +2,29 @@
 #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
 
 mod midibridge;
+mod webaudiobridge;
 use tokio::sync::mpsc;
 use tokio::sync::Mutex;
 
 fn main() {
-  let (async_input_transmitter, async_input_receiver) = mpsc::channel(1);
-  let (async_output_transmitter, async_output_receiver) = mpsc::channel(1);
+  let (async_input_transmitter_midi, async_input_receiver_midi) = mpsc::channel(1);
+  let (async_output_transmitter_midi, async_output_receiver_midi) = mpsc::channel(1);
+  let (async_input_transmitter_webaudio, async_input_receiver_webaudio) = mpsc::channel(1);
+  let (async_output_transmitter_webaudio, async_output_receiver_webaudio) = mpsc::channel(1);
   tauri::Builder
-    ::default()
-    .manage(midibridge::AsyncInputTransmit {
-      inner: Mutex::new(async_input_transmitter),
-    })
-    .invoke_handler(tauri::generate_handler![midibridge::sendmidi])
-    .setup(|_app| {
-      midibridge::init(async_input_receiver, async_output_receiver, async_output_transmitter);
-      Ok(())
-    })
-    .run(tauri::generate_context!())
-    .expect("error while running tauri application");
+  ::default()
+      .manage(midibridge::AsyncInputTransmit {
+        inner: Mutex::new(async_input_transmitter_midi),
+      })
+      .manage(webaudiobridge::AsyncInputTransmitWebAudio {
+        inner: Mutex::new(async_input_transmitter_webaudio),
+      })
+      .invoke_handler(tauri::generate_handler![midibridge::sendmidi, webaudiobridge::sendwebaudio])
+      .setup(|_app| {
+        midibridge::init(async_input_receiver_midi, async_output_receiver_midi, async_output_transmitter_midi);
+        webaudiobridge::init(async_input_receiver_webaudio, async_output_receiver_webaudio, async_output_transmitter_webaudio);
+        Ok(())
+      })
+      .run(tauri::generate_context!())
+      .expect("error while running tauri application");
 }

src-tauri/src/webaudiobridge.rs

@@ -0,0 +1,151 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+use midir::MidiOutput;
+use tokio::sync::{ mpsc, Mutex };
+use tokio::time::Instant;
+use serde::Deserialize;
+use std::thread::sleep;
+use web_audio_api::context::{AudioContext, AudioContextLatencyCategory, AudioContextOptions, BaseAudioContext};
+use web_audio_api::node::{AudioNode, AudioScheduledSourceNode, OscillatorType};
+
+pub struct WebAudioMessage {
+    pub note: f32,
+    pub instant: Instant,
+    pub offset: f64,
+    pub waveform: String,
+}
+
+pub struct AsyncInputTransmitWebAudio {
+    pub inner: Mutex<mpsc::Sender<Vec<WebAudioMessage>>>,
+}
+
+pub fn init(
+    async_input_receiver: mpsc::Receiver<Vec<WebAudioMessage>>,
+    mut async_output_receiver: mpsc::Receiver<Vec<WebAudioMessage>>,
+    async_output_transmitter: mpsc::Sender<Vec<WebAudioMessage>>
+) {
+    tauri::async_runtime::spawn(async move { async_process_model(async_input_receiver, async_output_transmitter).await });
+    let message_queue: Arc<Mutex<Vec<WebAudioMessage>>> = Arc::new(Mutex::new(Vec::new()));
+    /* ...........................................................
+           Listen For incoming messages and add to queue
+    ............................................................*/
+    let message_queue_clone = Arc::clone(&message_queue);
+    tauri::async_runtime::spawn(async move {
+        loop {
+            if let Some(package) = async_output_receiver.recv().await {
+                let mut message_queue = message_queue_clone.lock().await;
+                let messages = package;
+                for message in messages {
+                    (*message_queue).push(message);
+                }
+            }
+        }
+    });
+
+    let message_queue_clone = Arc::clone(&message_queue);
+    tauri::async_runtime::spawn(async move {
+        /* ...........................................................
+                            Prepare audio context
+        ............................................................*/
+        let latency_hint = match std::env::var("WEB_AUDIO_LATENCY").as_deref() {
+            Ok("playback") => AudioContextLatencyCategory::Playback,
+            _ => AudioContextLatencyCategory::default(),
+        };
+
+        let audio_context = AudioContext::new(AudioContextOptions {
+            latency_hint,
+            ..AudioContextOptions::default()
+        });
+
+        /* ...........................................................
+                            Process queued messages
+        ............................................................*/
+
+        loop {
+            let mut message_queue = message_queue_clone.lock().await;
+
+            //iterate over each message, play and remove messages when they are ready
+            message_queue.retain(|message| {
+                if message.instant.elapsed().as_millis() < message.offset as u128 {
+                    return true;
+                };
+
+                trigger_osc(&audio_context, message.note, message.waveform.clone());
+
+                return false;
+            });
+
+            sleep(Duration::from_millis(1));
+        }
+    });
+}
+
+pub async fn async_process_model(
+    mut input_reciever: mpsc::Receiver<Vec<WebAudioMessage>>,
+    output_transmitter: mpsc::Sender<Vec<WebAudioMessage>>
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    while let Some(input) = input_reciever.recv().await {
+        let output = input;
+        output_transmitter.send(output).await?;
+    }
+    Ok(())
+}
+#[derive(Deserialize)]
+pub struct MessageFromJS {
+    note: f32,
+    offset: f64,
+    waveform: String,
+}
+// Called from JS
+#[tauri::command]
+pub async fn sendwebaudio(
+    messagesfromjs: Vec<MessageFromJS>,
+    state: tauri::State<'_, AsyncInputTransmitWebAudio>
+) -> Result<(), String> {
+    let async_proc_input_tx = state.inner.lock().await;
+    let mut messages_to_process: Vec<WebAudioMessage> = Vec::new();
+
+    for m in messagesfromjs {
+        let message_to_process = WebAudioMessage {
+            note: m.note,
+            instant: Instant::now(),
+            offset: m.offset,
+            waveform: m.waveform,
+        };
+        messages_to_process.push(message_to_process);
+    }
+
+    async_proc_input_tx.send(messages_to_process).await.map_err(|e| e.to_string())
+}
+
+fn trigger_osc(audio_context: &AudioContext, freq: f32, osc_type: String) {
+    let env = audio_context.create_gain();
+    env.gain().set_value(0.);
+    env.connect(&audio_context.destination());
+
+    let osc = audio_context.create_oscillator();
+
+    match osc_type.as_str() {
+        "sine" => osc.set_type(OscillatorType::Sine),
+        "square" => osc.set_type(OscillatorType::Square),
+        "triangle" => osc.set_type(OscillatorType::Triangle),
+        "saw" => osc.set_type(OscillatorType::Sawtooth),
+        _ => osc.set_type(OscillatorType::Sine),
+    }
+
+    osc.connect(&env);
+
+    let now = audio_context.current_time();
+
+    let freq = freq;
+    osc.frequency().set_value(freq);
+
+    env.gain()
+        .set_value_at_time(0., now)
+        .linear_ramp_to_value_at_time(0.1, now + 0.01)
+        .exponential_ramp_to_value_at_time(0.0001, now + 2.);
+
+    osc.start_at(now);
+    osc.stop_at(now + 2.);
+}

website/src/repl/Repl.jsx

@@ -7,6 +7,7 @@ This program is free software: you can redistribute it and/or modify it under th
 import { cleanupDraw, cleanupUi, controls, evalScope, getDrawContext, logger } from '@strudel.cycles/core';
 import { CodeMirror, cx, flash, useHighlighting, useStrudel, useKeydown } from '@strudel.cycles/react';
 import { getAudioContext, initAudioOnFirstClick, resetLoadedSounds, webaudioOutput } from '@strudel.cycles/webaudio';
+import { webaudioDesktopOutput } from "@strudel/desktopbridge";
 import { createClient } from '@supabase/supabase-js';
 import { nanoid } from 'nanoid';
 import React, { createContext, useCallback, useEffect, useState, useMemo } from 'react';
@@ -124,7 +125,7 @@ export function Repl({ embedded = false }) {
   const { code, setCode, scheduler, evaluate, activateCode, isDirty, activeCode, pattern, started, stop, error } =
     useStrudel({
       initialCode: '// LOADING...',
-      defaultOutput: webaudioOutput,
+      defaultOutput: isTauri() ? webaudioDesktopOutput : webaudioOutput,
       getTime,
       beforeEval: async () => {
         setPending(true);