refactor: new clockbridge class to fix midi and osc timing

packages/midi/midi.mjs

@@ -21,7 +21,7 @@ import {
 } from '@strudel/core';
 import { noteToMidi, getControlName } from '@strudel/core';
 import { Note } from 'webmidi';
-import { getAudioContext } from '@strudel/webaudio';
+import { getAudioContext, getClockBridge } from '@strudel/webaudio';
 import { scheduleAtTime, ensureMinimalOutput } from '../superdough/helpers.mjs';
 import { getMidiDeviceNamesString, getDevice } from './util.mjs';
 import { MidiInput } from './input.mjs';
@@ -177,7 +177,11 @@ const isFirefox = navigator?.userAgent?.includes('Firefox');
 // firefox bug: https://bugzilla.mozilla.org/show_bug.cgi?id=2062997
 function timedSend(timeMs, fn) {
   if (isFirefox) {
-    const audioTime = getAudioContext().currentTime + (timeMs - performance.now()) / 1000;
+    const audioTime = getClockBridge().getAudioContextTime(timeMs);
+    if (!audioTime) {
+      logger('[midi]: skip event, not ready');
+      return;
+    }
     scheduleAtTime(() => fn(undefined), audioTime);
   } else {
     fn(timeMs);
@@ -281,11 +285,6 @@ function sendNote(note, velocity, duration, device, midichan, timeMs) {
   timedSend(timeMs + duration, (timeMs) => device.sendNoteOff(midiNote, { channels: midichan, time: timeMs }));
 }
 
-// thanks freya https://youtu.be/LSNQuFEDOyQ?si=ukZI2IGgWV_NDZzP&t=2979
-function expDecay(a, b, decay, dt) {
-  return b + (a - b) * Math.exp(-decay * dt);
-}
-
 /**
  * MIDI output: Opens a MIDI output port.
  * @tags external_io
@@ -343,26 +342,16 @@ Pattern.prototype.midi = function (midiport, options = {}) {
 
   ensureMinimalOutput();
 
-  let p; // filtered clock offset
-  let lastTime;
-
   return this.sortHapsByPart().onTrigger((hap, _currentTime, cps, targetTime) => {
     if (!WebMidi.enabled) {
       logger('Midi not enabled');
       return;
     }
-    const { contextTime, performanceTime } = getAudioContext().getOutputTimestamp();
-    if (!contextTime || !performanceTime) {
-      logger('[midi] skip midi event: not ready yet?');
+    const timeMs = getClockBridge().getPerformanceTime(targetTime);
+    if (!timeMs) {
+      logger('[midi] clockbridge not ready');
       return;
     }
-    // time conversion from audio context time (targetTime) to performance time (what midi needs)
-    const offset = performanceTime - contextTime * 1000; // clock offset in ms
-    const dt = performanceTime - (lastTime ?? performanceTime); // delta time since last midi hap
-    const decay = 1 / 10000; // how fast offset changes have an effect
-    p = expDecay(p ?? offset, offset, decay, dt); // smooth clock offset
-    lastTime = performanceTime;
-    const timeMs = targetTime * 1000 + p; // this is now correct in performance time
 
     hap.ensureObjectValue();
 

packages/osc/osc.mjs

@@ -4,7 +4,8 @@ Copyright (C) 2022 Strudel contributors - see <https://codeberg.org/uzu/strudel/
 This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
-import { logger, parseNumeral, register, isNote, noteToMidi, ClockCollator } from '@strudel/core';
+import { logger, parseNumeral, register, isNote, noteToMidi } from '@strudel/core';
+import { ensureMinimalOutput, getClockBridge } from '@strudel/webaudio';
 
 let connection; // Promise<OSC>
 function connect() {
@@ -54,13 +55,18 @@ export function parseControlsFromHap(hap, cps) {
   return controls;
 }
 
-const collator = new ClockCollator({});
-
 export async function oscTrigger(hap, currentTime, cps = 1, targetTime) {
   const ws = await connect();
+
+  const timeMs = getClockBridge().getPerformanceTime(targetTime);
+  if (!timeMs) {
+    logger('[osc] clockbridge not ready');
+    return;
+  }
+  const ts = performance.timeOrigin + timeMs;
+
   const controls = parseControlsFromHap(hap, cps);
   const keyvals = Object.entries(controls).flat();
-  const ts = collator.calculateTimestamp(currentTime, targetTime) * 1000;
   const msg = { address: '/dirt/play', args: keyvals, timestamp: ts };
 
   if ('oschost' in hap.value) {
@@ -82,4 +88,7 @@ export async function oscTrigger(hap, currentTime, cps = 1, targetTime) {
  * @memberof Pattern
  * @returns Pattern
  */
-export const osc = register('osc', (pat) => pat.onTrigger(oscTrigger));
+export const osc = register('osc', (pat) => {
+  ensureMinimalOutput();
+  return pat.onTrigger(oscTrigger);
+});

packages/osc/package.json

@@ -38,6 +38,7 @@
   "homepage": "https://codeberg.org/uzu/strudel#readme",
   "dependencies": {
     "@strudel/core": "workspace:*",
+    "@strudel/webaudio": "workspace:*",
     "osc": "^2.4.5",
     "ws": "^8.18.3"
   },

packages/superdough/audioContext.mjs

@@ -8,8 +8,9 @@ This program is free software: you can redistribute it and/or modify it under th
 */
 
 import { clearNodePool } from './nodePools.mjs';
+import { ClockBridge } from './clockbridge.mjs';
 
-let audioContext;
+let audioContext, clockBridge;
 
 export const setDefaultAudioContext = () => {
   return setAudioContext(new AudioContext());
@@ -23,6 +24,7 @@ export const setAudioContext = (context) => {
     audioContext.close();
   }
   audioContext = context;
+  clockBridge = new ClockBridge(audioContext);
   return audioContext;
 };
 
@@ -34,6 +36,13 @@ export const getAudioContext = () => {
   return audioContext;
 };
 
+export function getClockBridge() {
+  if (!clockBridge) {
+    getAudioContext(); // creates clockBridge
+  }
+  return clockBridge;
+}
+
 export function getAudioContextCurrentTime() {
   return getAudioContext().currentTime;
 }

packages/superdough/clockbridge.mjs

@@ -0,0 +1,41 @@
+/*
+clockbridge.mjs
+Copyright (C) 2022 Strudel contributors - see <https://codeberg.org/uzu/strudel/src/branch/main/packages/superdough/index.mjs>
+This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+// thanks freya https://youtu.be/LSNQuFEDOyQ?si=ukZI2IGgWV_NDZzP&t=2979
+export function expDecay(a, b, decay, dt) {
+  return b + (a - b) * Math.exp(-decay * dt);
+}
+
+// translates between audio and performance clock
+export class ClockBridge {
+  p; // smoothed clock offset
+  lastTime;
+  audioContext;
+  constructor(audioContext) {
+    this.audioContext = audioContext;
+  }
+  // delta between audio and performance time in ms
+  getOffset() {
+    const { contextTime, performanceTime } = this.audioContext.getOutputTimestamp();
+    if (!contextTime || !performanceTime) {
+      return;
+    }
+    const offset = performanceTime - contextTime * 1000; // clock offset in ms
+    const dt = performanceTime - (this.lastTime ?? performanceTime); // delta time since last hap
+    const decay = 1 / 10000; // how fast offset changes have an effect
+    this.p = expDecay(this.p ?? offset, offset, decay, dt); // smooth clock offset
+    this.lastTime = performanceTime;
+    return this.p;
+  }
+  getPerformanceTime(audioContextTime) {
+    const offset = this.getOffset();
+    return audioContextTime * 1000 + offset; // this is now correct in performance time (ms)
+  }
+  getAudioContextTime(performanceTime) {
+    const offset = this.getOffset();
+    return (performanceTime - offset) / 1000; // this is now correct in audio context time (seconds)
+  }
+}

packages/superdough/index.mjs

@@ -14,3 +14,4 @@ export * from './modulators.mjs';
 export * from './dspworklet.mjs';
 export * from './audioContext.mjs';
 export * from './wavetable.mjs';
+export * from './clockbridge.mjs';

pnpm-lock.yaml

@@ -470,6 +470,9 @@ importers:
       '@strudel/core':
         specifier: workspace:*
         version: link:../core
+      '@strudel/webaudio':
+        specifier: workspace:*
+        version: link:../webaudio
       osc:
         specifier: ^2.4.5
         version: 2.4.5