mondo highlighting

packages/mondo/mondo.mjs

@@ -19,35 +19,46 @@ export class MondoParser {
     pipe: /^\./,
     stack: /^,/,
     op: /^[*/]/,
-    plain: /^[a-zA-Z0-9-_]+/,
+    plain: /^[a-zA-Z0-9-_\^]+/,
   };
-  constructor(config) {
-    this.config = config;
-  }
   // matches next token
-  next_token(code) {
+  next_token(code, offset = 0) {
     for (let type in this.token_types) {
       const match = code.match(this.token_types[type]);
       if (match) {
-        return { type, value: match[0] };
+        let token = { type, value: match[0] };
+        if (offset !== -1) {
+          // add location
+          token.loc = [offset, offset + match[0].length];
+        }
+        return token;
       }
     }
     throw new Error(`mondo: could not match '${code}'`);
   }
   // takes code string, returns list of matched tokens (if valid)
-  tokenize(code) {
+  tokenize(code, offset = 0) {
     let tokens = [];
+    let locEnabled = offset !== -1;
+    let trim = () => {
+      // trim whitespace at start, update offset
+      offset += code.length - code.trimStart().length;
+      // trim start and end to not confuse parser
+      return code.trim();
+    };
+    code = trim();
     while (code.length > 0) {
-      code = code.trim();
-      const token = this.next_token(code);
+      code = trim();
+      const token = this.next_token(code, locEnabled ? offset : -1);
       code = code.slice(token.value.length);
+      offset += token.value.length;
       tokens.push(token);
     }
     return tokens;
   }
   // take code, return abstract syntax tree
-  parse(code) {
-    this.tokens = this.tokenize(code);
+  parse(code, offset) {
+    this.tokens = this.tokenize(code, offset);
     const expressions = [];
     while (this.tokens.length) {
       expressions.push(this.parse_expr());
@@ -244,6 +255,20 @@ export class MondoParser {
     }
     return token;
   }
+  get_locations(code, offset = 0) {
+    let walk = (ast, locations = []) => {
+      if (ast.type === 'list') {
+        return ast.children.slice(1).forEach((child) => walk(child, locations));
+      }
+      if (ast.loc) {
+        locations.push(ast.loc);
+      }
+    };
+    const ast = this.parse(code, offset);
+    let locations = [];
+    walk(ast, locations);
+    return locations;
+  }
 }
 
 export function printAst(ast, compact = false, lvl = 0) {
@@ -259,10 +284,9 @@ export function printAst(ast, compact = false, lvl = 0) {
 
 // lisp runner
 export class MondoRunner {
-  constructor(lib, config = {}) {
-    this.parser = new MondoParser(config);
+  constructor(lib) {
+    this.parser = new MondoParser();
     this.lib = lib;
-    this.config = config;
   }
   // a helper to check conditions and throw if they are not met
   assert(condition, error) {
@@ -270,9 +294,9 @@ export class MondoRunner {
       throw new Error(error);
     }
   }
-  run(code) {
-    const ast = this.parser.parse(code);
-    console.log(printAst(ast));
+  run(code, offset = 0) {
+    const ast = this.parser.parse(code, offset);
+    // console.log(printAst(ast));
     return this.call(ast);
   }
   call(ast) {
@@ -284,13 +308,13 @@ export class MondoRunner {
     // process args
     const args = ast.children.slice(1).map((arg) => {
       if (arg.type === 'string') {
-        return this.lib.string(arg.value.slice(1, -1));
+        return this.lib.string(arg.value.slice(1, -1), arg);
       }
       if (arg.type === 'plain') {
-        return this.lib.plain(arg.value);
+        return this.lib.plain(arg.value, arg);
       }
       if (arg.type === 'number') {
-        return this.lib.number(Number(arg.value));
+        return this.lib.number(Number(arg.value), arg);
       }
       return this.call(arg);
     });

packages/mondo/test/mondo.test.mjs

@@ -8,8 +8,24 @@ import { describe, expect, it } from 'vitest';
 import { MondoParser, MondoRunner, printAst } from '../mondo.mjs';
 
 const parser = new MondoParser();
-const p = (code) => parser.parse(code);
+const p = (code) => parser.parse(code, -1);
 
+describe('mondo tokenizer', () => {
+  const parser = new MondoParser();
+  it('should tokenize with locations', () =>
+    expect(
+      parser
+        .tokenize('(one two three)')
+        .map((t) => t.value + '=' + t.loc.join('-'))
+        .join(' '),
+    ).toEqual('(=0-1 one=1-4 two=5-8 three=9-14 )=14-15'));
+  // it('should parse with locations', () => expect(parser.parse('(one two three)')).toEqual());
+  it('should get locations', () =>
+    expect(parser.get_locations('s bd rim')).toEqual([
+      [2, 4],
+      [5, 8],
+    ]));
+});
 describe('mondo s-expressions parser', () => {
   it('should parse an empty string', () => expect(p('')).toEqual({ type: 'list', children: [] }));
   it('should parse a single item', () =>

packages/mondough/mondough.mjs

@@ -1,25 +1,22 @@
-import { strudelScope, reify, fast, slow, isControlName } from '@strudel/core';
+import { strudelScope, reify, fast, slow } from '@strudel/core';
+import { registerLanguage } from '@strudel/transpiler';
 import { MondoRunner } from '../mondo/mondo.mjs';
 
-let runner = new MondoRunner(strudelScope, { pipepost: true });
+let runner = new MondoRunner(strudelScope, { pipepost: true, loc: true });
 
-//strudelScope.plain = reify;
-strudelScope.plain = (v) => {
-  // console.log('plain', v);
-  return reify(v);
-  // return v;
+let getLeaf = (value, token) => {
+  const [from, to] = token.loc;
+  return reify(value).withLoc(from, to);
 };
-// strudelScope.number = (n) => n;
-strudelScope.number = reify;
+
+strudelScope.plain = getLeaf;
+strudelScope.number = getLeaf;
 
 strudelScope.call = (fn, args, name) => {
   const [pat, ...rest] = args;
   if (!['seq', 'cat', 'stack'].includes(name)) {
     args = [...rest, pat];
   }
-
-  // console.log('call', name, ...flipped);
-
   return fn(...args);
 };
 
@@ -30,6 +27,12 @@ export function mondo(code, offset = 0) {
   if (Array.isArray(code)) {
     code = code.join('');
   }
-  const pat = runner.run(code, { pipepost: true });
+  const pat = runner.run(code, offset);
   return pat;
 }
+
+// tell transpiler how to get locations for mondo`` calls
+registerLanguage('mondo', {
+  getLocations: (code, offset) => runner.parser.get_locations(code, offset),
+});
+

packages/mondough/package.json

@@ -32,7 +32,8 @@
   },
   "homepage": "https://github.com/tidalcycles/strudel#readme",
   "dependencies": {
-    "@strudel/core": "workspace:*"
+    "@strudel/core": "workspace:*",
+    "@strudel/transpiler": "workspace:*"
   },
   "devDependencies": {
     "mondo": "*",

packages/transpiler/transpiler.mjs

@@ -8,6 +8,16 @@ export function registerWidgetType(type) {
   widgetMethods.push(type);
 }
 
+let languages = new Map();
+// config = { getLocations: (code: string, offset?: number) => number[][] }
+// see mondough.mjs for example use
+// the language will kick in when the code contains a template literal of type
+// example: mondo`...` will use language of type "mondo"
+// TODO: refactor tidal.mjs to use this
+export function registerLanguage(type, config) {
+  languages.set(type, config);
+}
+
 export function transpiler(input, options = {}) {
   const { wrapAsync = false, addReturn = true, emitMiniLocations = true, emitWidgets = true } = options;
 
@@ -26,7 +36,19 @@ export function transpiler(input, options = {}) {
 
   walk(ast, {
     enter(node, parent /* , prop, index */) {
-      if (isTidalTeplateLiteral(node)) {
+      if (isLanguageLiteral(node)) {
+        const { name } = node.tag;
+        const language = languages.get(name);
+        const code = node.quasi.quasis[0].value.raw;
+        const offset = node.quasi.start + 1;
+        if (emitMiniLocations) {
+          const locs = language.getLocations(code, offset);
+          miniLocations = miniLocations.concat(locs);
+        }
+        this.skip();
+        return this.replace(languageWithLocation(name, code, offset));
+      }
+      if (isTemplateLiteral(node, 'tidal')) {
         const raw = node.quasi.quasis[0].value.raw;
         const offset = node.quasi.start + 1;
         if (emitMiniLocations) {
@@ -219,12 +241,16 @@ function labelToP(node) {
   };
 }
 
+function isLanguageLiteral(node) {
+  return node.type === 'TaggedTemplateExpression' && languages.has(node.tag.name);
+}
+
 // tidal highlighting
 // this feels kind of stupid, when we also know the location inside the string op (tidal.mjs)
 // but maybe it's the only way
 
-function isTidalTeplateLiteral(node) {
-  return node.type === 'TaggedTemplateExpression' && node.tag.name === 'tidal';
+function isTemplateLiteral(node, value) {
+  return node.type === 'TaggedTemplateExpression' && node.tag.name === value;
 }
 
 function collectHaskellMiniLocations(haskellCode, offset) {
@@ -262,3 +288,33 @@ function tidalWithLocation(value, offset) {
     optional: false,
   };
 }
+
+function mondoWithLocation(value, offset) {
+  return {
+    type: 'CallExpression',
+    callee: {
+      type: 'Identifier',
+      name: 'mondo',
+    },
+    arguments: [
+      { type: 'Literal', value },
+      { type: 'Literal', value: offset },
+    ],
+    optional: false,
+  };
+}
+
+function languageWithLocation(name, value, offset) {
+  return {
+    type: 'CallExpression',
+    callee: {
+      type: 'Identifier',
+      name: name,
+    },
+    arguments: [
+      { type: 'Literal', value },
+      { type: 'Literal', value: offset },
+    ],
+    optional: false,
+  };
+}