From 24b69ad1bcc6861268f34f4cf01685e8f11bfeca Mon Sep 17 00:00:00 2001 From: jeromew Date: Sun, 30 Nov 2025 14:09:03 +0000 Subject: [PATCH] [perf] Add audiograph `await debugAudiograph();` tool --- website/src/repl/audiograph.mjs | 618 ++++++++++++++++++++++++++++ website/src/repl/useReplContext.jsx | 2 + 2 files changed, 620 insertions(+) create mode 100644 website/src/repl/audiograph.mjs diff --git a/website/src/repl/audiograph.mjs b/website/src/repl/audiograph.mjs new file mode 100644 index 000000000..d203496ae --- /dev/null +++ b/website/src/repl/audiograph.mjs @@ -0,0 +1,618 @@ +/* +audiograph.mjs - show a svg view of the web audio API graph built during a playback +Copyright (C) 2025 Strudel contributors - see +This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . +*/ + +// main entry point is `debugAudiograph` + +import { logger } from '@strudel/core'; +import { getAudioContext, getSuperdoughAudioController, webaudioOutput } from '@strudel/webaudio'; + +let mermaid = null; +let svgPanZoom = null; + +let running = false; +let hap_count = 0; + +let cache = new Map(); +const initCache = JSON.stringify({ + connect: [], + where: [], + disconnectAll: 0, + disconnectOne: 0, + stopCount: 0, + ac: null, + creation: null, +}); + +let toggleOrig; + +function stackTrace() { + var err = new Error(); + const stacktrace = err.stack; + const lines = stacktrace.split('\n'); + let lineIndex = lines.findIndex((line) => line !== 'Error' && !line.includes('audiograph.mjs')); + if (lines[lineIndex].includes('gainNode')) lineIndex++; + if (lines[lineIndex].includes('getWorklet')) lineIndex++; + const line = lines[lineIndex].replace(/\s*at\s/, '').replace('http', '@'); + let match; + match = line.match(/([^@]*)@.*packages(\/[^:]+:\d+:\d+)/); + if (match) { + return match[1].replace(/[^.:/a-zA-Z0-9]/g, '') + '@' + match[2].replace(/[^.:/a-zA-Z0-9]/g, ''); + } + return '@'; +} + +// This captures all AudioNodes lazily +// when an `.audioid` property is called +// no solution was found to hook +// AudioNode's constructor directly +let audioid = 0; +const lazyRegister = (o) => { + Object.defineProperty(o.prototype, 'audioid', { + get: function () { + if (!this._audioid) { + this._audioid = ++audioid; + const s = JSON.parse(initCache); + s.type = this.constructor.name === 'AudiographNode' ? this.constructor._parentClassName : this.constructor.name; + s.ac = this.context?.constructor.name || 'AudioParam'; + s.creation = s.creation || stackTrace(); + cache.set(this._audioid, s); + } + return this._audioid; + }, + enumerable: false, + configurable: true, + }); +}; + +// extend a specific AudioNode's constructor +// necessary when creation is done direclty by +// calling the constructor +// eg: new GainNode(...) +const audioNodeHook = (node) => { + const name = node.prototype.constructor.name; + const PatchedNode = class AudiographNode extends node { + constructor(...args) { + super(...args); + // trigger the lazy register + this._audioid = this.audioid; + } + }; + PatchedNode._parentClassName = name; + window[name] = PatchedNode; +}; + +const drawMessage = async function (message) { + const element = document.querySelector('.strudel-mermaid'); + let gd = ''; + gd += '---\n'; + gd += 'config:\n'; + gd += ' flowchart:\n'; + gd += ' wrappingWidth: 600\n'; + gd += '---\n'; + gd += 'flowchart LR\n'; + gd += 'id[' + message.replaceAll(' ', ' ') + ']\n'; + + let { svg } = await mermaid.render('strudelSvgId', gd); + svg = svg.replace(/max-width:\s[0-9.]*px;/i, 'height: 100%'); + svg = svg.replaceAll('&nbsp;', ' '); + element.innerHTML = svg; +}; + +const drawDiagram = async function () { + const element = document.querySelector('.strudel-mermaid'); + + let code = window.strudelMirror.code; + code = code.replace(/^await debugAudiograph.*\n?/gm, ''); + code = '// date: ' + new Date().toISOString() + '\n\n' + code; + code = '// host: ' + document.location.hostname + '\n' + code; + const codeLines = code.split(/(?:\n|\r\n?)/); + const maxLineLength = codeLines.reduce((memo, line) => Math.max(memo, line.length), 0); + + // https://mermaid.js.org/syntax/flowchart.html + let gd = ''; + gd += '---\n'; + gd += 'config:\n'; + gd += ' flowchart:\n'; + gd += ' wrappingWidth: ' + 14 * maxLineLength + '\n'; + gd += '---\n'; + gd += 'flowchart TB\n'; + gd += '\tsubgraph AG[STRUDEL AUDIOGRAPH]\n'; + + // seed graph builder with all + // unconnected nodes + let lookup = []; + cache.forEach((v, k) => { + if (v.connect.length === 0) lookup.push(k); + }); + const relations = []; + let curRelations; + const zombieCount = 0; + const sourceLoc = (stack) => { + if (stack === '@') return stack; + return stack.replace('@', '\n').replace('/superdough/', '/'); + }; + const label = (s) => { + const source = s.creation ? '\n' + sourceLoc(s.creation) : ''; + let lb = '[' + '**' + s.type + '**' + source + ']'; + if (s.ac === 'OfflineAudioContext') lb = '[' + lb + ']'; + return lb; + }; + const isLeak = (s) => { + return ( + ['AudioDestinationNode', 'AudioParam'].indexOf(s.type) === -1 && + s.disconnectAll === 0 && + s.connect.length > s.disconnectOne + ); + }; + do { + curRelations = relations.length; + lookup.slice().forEach((n) => { + cache.forEach((v, k) => { + if (v.connect.indexOf(n) !== -1) { + if (lookup.indexOf(k) === -1) lookup.push(k); + gd += v.connect + .map((i) => { + if (lookup.indexOf(i) === -1) lookup.push(i); + if (relations.indexOf(k + '-' + i) === -1) { + relations.push(k + '-' + i); + return ( + '\t\tnode' + + k + + label(v) + + ' -- ' + + sourceLoc(v.where[0]) + + ' --> node' + + i + + label(cache.get(i)) + + '\n' + ); + } + }) + .join(''); + } + if (k === n) { + gd += v.connect + .map((i) => { + if (lookup.indexOf(i) === -1) lookup.push(i); + if (relations.indexOf(k + '-' + i) === -1) { + relations.push(k + '-' + i); + return ( + '\t\tnode' + + k + + label(v) + + ' -- ' + + sourceLoc(v.where[0]) + + ' --> node' + + i + + label(cache.get(i)) + + '\n' + ); + } + }) + .join(''); + } + }); + }); + } while (relations.length > curRelations /*&& lookup.length < 100*/); + + const codePlaceholder = 'm'.repeat(maxLineLength); + gd += '\tsubgraph LEGEND\n'; + gd += '\t\tlegend1[in AudioContext]\n'; + gd += '\t\tlegend2[[in OfflineAudioContext]]\n'; + gd += '\t\tlegend3[not disconnected]\n'; + gd += '\t\tlegend4[AudioParam]\n'; + gd += '\t\tlegend5[AudioDestinationNode]\n'; + gd += '\tend\n'; + gd += '\tsubgraph CODE[Strudel Code]\n'; + // we use a codePlaceholder to + // - avoid problems with special chars + // - stop mermaid to split lines on space with multiple tspans + // - force mermaid to prepare a sufficiently sized zone + gd += '\ncode[' + (codePlaceholder + '
').repeat(codeLines.length) + ']\n'; + gd += '\tend\n'; + gd += '\tend\n'; + gd += '\tclassDef audioparam fill:#6f6;\n'; + gd += '\tclassDef destination fill:#99f;\n'; + gd += '\tclassDef suspect fill:#f96,stroke:#f00,stroke-width:2px;\n'; + gd += '\tclass legend3 suspect;\n'; + gd += '\tclass legend4 audioparam;\n'; + gd += '\tclass legend5 destination;\n'; + cache.forEach((v, k) => { + if (isLeak(v)) { + gd += '\tclass node' + k + ' suspect;\n'; + } + if (v.type === 'AudioParam') { + gd += '\tclass node' + k + ' audioparam;\n'; + } + if (v.type === 'AudioDestinationNode') { + gd += '\tclass node' + k + ' destination;\n'; + } + }); + let { svg } = await mermaid.render('strudelSvgId', gd); + + // put real code in code zone + let idx = 0; + const escapeHtml = (unsafe) => { + return unsafe + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); + }; + svg = svg.replaceAll(codePlaceholder, () => escapeHtml(codeLines[idx++])); + + // improve sizing on web page + svg = svg.replace(/max-width:\s[0-9.]*px;/i, 'height: 100%'); + element.innerHTML = svg; + + // align the code lines + let svgText = document.querySelector('[id^=flowchart-code] text'); + svgText.setAttributeNS(null, 'style', 'text-anchor: start;'); + const svgElement = document.querySelector('svg'); + const svgLabel = document.querySelector('svg [id^=flowchart-code] .label'); + const transformList = svgLabel.transform.baseVal; + const svgTransform = svgElement.createSVGTransform(); + const tspans = Array.from(document.querySelectorAll('[id^=flowchart-code] tspan.text-inner-tspan')); + let tspansMaxLength = tspans.reduce((memo, tspan) => Math.max(memo, tspan.getComputedTextLength()), 0); + svgTransform.setTranslate(-tspansMaxLength / 2, 0); + transformList.appendItem(svgTransform); + + let doPan = false; + let eventsHandler; + let panZoom; + let mousepos; + + eventsHandler = { + haltEventListeners: ['mousedown', 'mousemove', 'mouseup'], + mouseDownHandler: function (ev) { + if (event.target.className == '[object SVGAnimatedString]') { + doPan = true; + mousepos = { + x: ev.clientX, + y: ev.clientY, + }; + } + }, + mouseMoveHandler: function (ev) { + if (doPan) { + panZoom.panBy({ + x: ev.clientX - mousepos.x, + y: ev.clientY - mousepos.y, + }); + mousepos = { + x: ev.clientX, + y: ev.clientY, + }; + window.getSelection().removeAllRanges(); + } + }, + mouseUpHandler: function (ev) { + doPan = false; + }, + init: function (options) { + options.svgElement.addEventListener('mousedown', this.mouseDownHandler, false); + options.svgElement.addEventListener('mousemove', this.mouseMoveHandler, false); + options.svgElement.addEventListener('mouseup', this.mouseUpHandler, false); + }, + destroy: function (options) { + options.svgElement.removeEventListener('mousedown', this.mouseDownHandler, false); + options.svgElement.removeEventListener('mousemove', this.mouseMoveHandler, false); + options.svgElement.removeEventListener('mouseup', this.mouseUpHandler, false); + }, + }; + panZoom = svgPanZoom('#strudelSvgId', { + zoomEnabled: true, + controlIconsEnabled: true, + fit: 1, + center: 1, + zoomScaleSensitivity: 0.4, + customEventsHandler: eventsHandler, + }); +}; + +const svgExport = async () => { + const a = document.createElement('a'); + document.body.appendChild(a); + a.style = 'display: none'; + + const selector = '.strudel-mermaid'; + const bbox = document.querySelector('svg g').getBBox(); + let transform, style; + // clean pan-zoom viewport + const pzViewport = document.querySelector('.svg-pan-zoom_viewport'); + if (pzViewport) { + transform = pzViewport.transform; + style = pzViewport.style; + pzViewport.setAttribute('transform', ''); + pzViewport.style = ''; + } + + const spzMin = await fetch('https://cdn.jsdelivr.net/npm/svg-pan-zoom@3.6.2/dist/svg-pan-zoom.min.js').then((res) => + res.text(), + ); + + const scriptContent = ''; + // prepare svg + const content = document + .querySelector(selector) + .innerHTML.replaceAll('
', '
') + // remove useless tags + .replace(/ { + if (k <= audioid) cache.delete(k); + }); +}; + +const postProcessing = async function () { + hap_count = 0; + await drawDiagram(); + + resetAudioOutput(audioid); +}; + +const defaultOptions = { + StopAfterHapCount: 10, + hapsBatch: 0, + maxEdges: 10000, + maxTextSize: 200000, + audioAPIBreathingRoomSec: 5, +}; + +// `StopAfterHapCount` : +// The player will auto-stop after hap count have +// been played. when StopAfterHapCount = 0, it will +// continue playing until 'stop' is clicked +// `audioAPIBreathingRoomSec` : +// how much time should we wait after 'stop' to let +// the audioAPI finish its tail of ondended calls +// `hapsBatch` : +// the AudioGraph will be displayed every hapsBatch haps +// when hapsBatch = 0, AudioGraph will only be displayed +// after and auto-stop or after 'stop' is clicked +// In hapsBatch mode you will probably see a trailing of +// non disconnected notes on the graph because the audio +// API may have some lag disconnecting them +// cf also audioAPIBreathingRoomSec +// `maxEdges` +// This is a mermaid.js config that forces a hard limit +// on the maximum number of Edges of a graph +// needs a reload to be taken into account +// `maxTextSize` +// This is a mermaid.js config that forces a hard limit +// on the maximum text size of a graph definition +// needs a reload to be taken into account + +export const debugAudiograph = async (argOptions = {}) => { + const options = Object.assign({}, defaultOptions, argOptions); + const { StopAfterHapCount, hapsBatch, maxEdges, maxTextSize, audioAPIBreathingRoomSec } = options; + const sm = window.strudelMirror; + const code = sm.code; + if (!code.match(/await\s+debugAudiograph/)) { + throw new Error('you need to call `await debugAudiograph()` for audiograph to work'); + } + const emptyOptions = /await\s+debugAudiograph\(\)/.exec(code); + if (emptyOptions) { + const cutCode = emptyOptions.index + emptyOptions[0].length - 1; + const codeOptions = JSON.stringify({ StopAfterHapCount: StopAfterHapCount }).replaceAll('"', ''); + sm.setCode(code.slice(0, cutCode) + codeOptions + code.slice(cutCode)); + } + + if (window.audiograph === undefined) { + const ag = (window.audiograph = {}); + + toggleOrig = sm.toggle; + + //////////////////////////////////////// + // step 1: web audio api instrumentation + //////////////////////////////////////// + + // path AudioNode & AudioParam + // to give them lazy ids + // this captures both `ac.createGain` + // and `new GainNode(..)` patterns + lazyRegister(AudioNode); + lazyRegister(AudioParam); + + const audioNodes = [ + AudioBufferSourceNode, + AudioWorkletNode, + AnalyserNode, + BiquadFilterNode, + ChannelMergerNode, + ChannelSplitterNode, + ConstantSourceNode, + ConvolverNode, + DelayNode, + DynamicsCompressorNode, + GainNode, + IIRFilterNode, + OscillatorNode, + PannerNode, + StereoPannerNode, + WaveShaperNode, + ]; + audioNodes.map((n) => audioNodeHook(n)); + + // patch BaseAudioContext factory methods + // to capture the source reference + Object.getOwnPropertyNames(BaseAudioContext.prototype) + .filter((n) => n.startsWith('create') && ['createBuffer'].indexOf(n) === -1) + .map((name) => { + const orig = BaseAudioContext.prototype[name]; + BaseAudioContext.prototype[name] = function (...args) { + const result = orig.call(this, ...args); + const s = cache.get(result.audioid); + s.creation = stackTrace(); + return result; + }; + }); + + const connectOrig = AudioNode.prototype.connect; + AudioNode.prototype.connect = function (destination, ...args) { + const result = connectOrig.call(this, destination, ...args); + const s = cache.get(this.audioid); + s.connect.push(destination.audioid); + s.where.push(stackTrace()); + return result; + }; + + const disconnectOrig = AudioNode.prototype.disconnect; + AudioNode.prototype.disconnect = function (destination, ...args) { + const result = disconnectOrig.call(this, destination, ...args); + const s = cache.get(this.audioid); + if (s.connect.length) { + if (destination) { + s.disconnectOne++; + } else { + s.disconnectAll++; + } + } else { + logger('WEIRD: node ' + this.audioid + 'called disconnect before any call to connect !'); + logger(new Error().stack); + console.log(cache); + } + return result; + }; + + // call reset 2 times to handle reload + 'play' + // the first reset's disconnect adds audioid tags on previous outputs + // that were not tagged (wrong cutoff) + resetAudioOutput(audioid); + // the second reset has the correct audioid cutoff + resetAudioOutput(audioid); + + //////////////////////////////////////// + // step 2: Load external modules + //////////////////////////////////////// + + const { default: mermaidModule } = await import( + 'https://cdn.jsdelivr.net/npm/mermaid@11.12.1/dist/mermaid.esm.mjs' + ); + mermaid = mermaidModule; + + mermaid.initialize({ + startOnLoad: false, + themeCSS: '.flowchart { height: 100%; }', + maxEdges: maxEdges, + maxTextSize: maxTextSize, + htmlLabels: false, + flowchart: { + htmlLabels: false, + }, + }); + + const { default: svgPanZoomModule } = await import('https://esm.sh/svg-pan-zoom'); + svgPanZoom = svgPanZoomModule; + + ////////////////////////////////////////// + // step 3: UI modifications + ////////////////////////////////////////// + + // add audiograph panel + if (!document.querySelector('.strudel-mermaid')) { + const mermaidDiv = document.createElement('div'); + mermaidDiv.className = 'strudel-mermaid'; + mermaidDiv.style = 'min-height: 600px; width: 60%'; + const referenceNode = document.querySelector('#code'); + referenceNode.parentNode.insertBefore(mermaidDiv, referenceNode.nextSibling); + } + + // add svg export button + if (!document.querySelector('button[title=svg]')) { + const exportButton = document.createElement('button'); + exportButton.innerHTML = 'ExportDiagram'; + exportButton.title = 'svg'; + exportButton.onclick = svgExport; + const updateButton = document.querySelector('button[title=update]'); + updateButton.parentNode.insertBefore(exportButton, updateButton); + } + } + + if (!running) { + running = true; + } + + if (hapsBatch === 0 || hap_count < hapsBatch) { + let msg = ''; + msg += 'Recording activity...'; + msg += '\npress stop to build diagram'; + if (StopAfterHapCount) { + msg += '\nwill stop automatically in ' + Math.max(StopAfterHapCount - hap_count, 0) + ' haps'; + } + await drawMessage(msg); + } + + sm.toggle = async () => { + running = false; + sm.toggle = toggleOrig; + // schedule `toggle` on the js main loop + // to avoid interfering with any on-flight onTick + // not doing this can lead to a phase > 0 which will + // break the next start + setTimeout(sm.toggle.bind(sm), 0); + await drawMessage('please wait ' + audioAPIBreathingRoomSec + ' seconds\n' + 'the audio API is finishing its work'); + setTimeout(postProcessing, audioAPIBreathingRoomSec * 1000); + }; + + /*global all*/ + all((pat) => + pat.onTrigger(async (hap, duration, cps, t) => { + hap_count++; + const key = Object.entries(hap.value) + .map((param) => param.join('/')) + .join('/'); + + // if we reached StopAfterHapCount, click 'stop' + if (StopAfterHapCount && hap_count > StopAfterHapCount) { + if (running) { + await sm.toggle(); + } + // stop sending haps to superdough(...) + return; + } + + await webaudioOutput(hap, t, hap.duration / cps, cps, t); + + if (hapsBatch && hap_count % hapsBatch === 0) drawDiagram(); + }), + ); +}; diff --git a/website/src/repl/useReplContext.jsx b/website/src/repl/useReplContext.jsx index 8dcb6b754..7ef86c08e 100644 --- a/website/src/repl/useReplContext.jsx +++ b/website/src/repl/useReplContext.jsx @@ -36,6 +36,7 @@ import { getRandomTune, initCode, loadModules, shareCode } from './util.mjs'; import './Repl.css'; import { setInterval, clearInterval } from 'worker-timers'; import { getMetadata } from '../metadata_parser'; +import { debugAudiograph } from './audiograph'; const { latestCode, maxPolyphony, audioDeviceName, multiChannelOrbits } = settingsMap.get(); let modulesLoading, presets, drawContext, clearCanvas, audioReady; @@ -129,6 +130,7 @@ export function useReplContext() { bgFill: false, }); window.strudelMirror = editor; + window.debugAudiograph = debugAudiograph; // init settings initCode().then(async (decoded) => {