Merge pull request #1124 from daslyfe/bug_sampledb

Fix indexDB failing with large amount of files

CONTRIBUTING.md

@@ -130,20 +130,14 @@ npx lerna version --no-private
 # publish all packages inside /packages using pnpm! don't use lerna to publish!!
 pnpm --filter "./packages/**" publish --dry-run
 
-# the last command was only a dry-run, make sure everything looks ok, if yes, run the same command without flag
+# the last command was only a dry-run. if everything looks ok, run this:
+
+pnpm --filter "./packages/**" publish --access public
 ```
 
 To manually publish a single package, increase the version in the `package.json`, then run `pnpm publish`.
 Important: Always publish with `pnpm`, as `npm` does not support overriding main files in `publishConfig`, which is done in all the packages.
 
-### New Packages
-
-To add a new package, you have to publish it manually the first time, using:
-
-```sh
-cd packages/<package-name> && pnpm publish --access public
-```
-
 ## Have Fun
 
 Remember to have fun, and that this project is driven by the passion of volunteers!

website/src/repl/files.mjs

@@ -75,7 +75,8 @@ export const walkFileTree = (node, fn) => {
   }
 };
 
-export const isAudioFile = (filename) => ['wav', 'mp3'].includes(filename.split('.').slice(-1)[0]);
+export const isAudioFile = (filename) =>
+  ['wav', 'mp3', 'flac', 'ogg', 'm4a', 'aac'].includes(filename.split('.').slice(-1)[0]);
 
 function uint8ArrayToDataURL(uint8Array) {
   const blob = new Blob([uint8Array], { type: 'audio/*' });

website/src/repl/idbutils.mjs

@@ -12,7 +12,7 @@ export const userSamplesDBConfig = {
 };
 
 // deletes all of the databases, useful for debugging
-const clearIDB = () => {
+function clearIDB() {
   window.indexedDB
     .databases()
     .then((r) => {
@@ -21,37 +21,59 @@ const clearIDB = () => {
     .then(() => {
       alert('All data cleared.');
     });
-};
+}
 
 // queries the DB, and registers the sounds so they can be played
-export const registerSamplesFromDB = (config = userSamplesDBConfig, onComplete = () => {}) => {
+export function registerSamplesFromDB(config = userSamplesDBConfig, onComplete = () => {}) {
   openDB(config, (objectStore) => {
-    let query = objectStore.getAll();
+    const query = objectStore.getAll();
+    query.onerror = (e) => {
+      logger('User Samples failed to load ', 'error');
+      onComplete();
+      console.error(e?.target?.error);
+    };
+
     query.onsuccess = (event) => {
       const soundFiles = event.target.result;
       if (!soundFiles?.length) {
         return;
       }
       const sounds = new Map();
+
+      Promise.all(
         [...soundFiles]
           .sort((a, b) => a.title.localeCompare(b.title, undefined, { numeric: true, sensitivity: 'base' }))
-        .forEach((soundFile) => {
+          .map((soundFile, i) => {
             const title = soundFile.title;
             if (!isAudioFile(title)) {
               return;
             }
             const splitRelativePath = soundFile.id.split('/');
-          const parentDirectory =
+            let parentDirectory =
               //fallback to file name before period and seperator if no parent directory
               splitRelativePath[splitRelativePath.length - 2] ?? soundFile.id.split(/\W+/)[0] ?? 'user';
-          const soundPath = soundFile.blob;
+            const blob = soundFile.blob;
+
+            // Files used to be uploaded as base64 strings, After Jan 1 2025 this check can be safely deleted
+            if (typeof blob === 'string') {
+              const soundPaths = sounds.get(parentDirectory) ?? new Set();
+              soundPaths.add(blob);
+              sounds.set(parentDirectory, soundPaths);
+              return;
+            }
+
+            return blobToDataUrl(blob).then((soundPath) => {
               const soundPaths = sounds.get(parentDirectory) ?? new Set();
               soundPaths.add(soundPath);
               sounds.set(parentDirectory, soundPaths);
+              return;
             });
-
+          }),
+      )
+        .then(() => {
           sounds.forEach((soundPaths, key) => {
             const value = Array.from(soundPaths);
+
             registerSound(key, (t, hapValue, onended) => onTriggerSample(t, hapValue, onended, value), {
               type: 'sample',
               samples: value,
@@ -60,15 +82,20 @@ export const registerSamplesFromDB = (config = userSamplesDBConfig, onComplete =
               tag: undefined,
             });
           });
+
           logger('imported sounds registered!', 'success');
           onComplete();
+        })
+        .catch((error) => {
+          logger('Something went wrong while registering saved samples from the index db', 'error');
+          console.error(error);
+        });
     };
   });
-};
+}
-// creates a blob from a buffer that can be read
+
-async function bufferToDataUrl(buf) {
+async function blobToDataUrl(blob) {
   return new Promise((resolve) => {
-    var blob = new Blob([buf], { type: 'application/octet-binary' });
     var reader = new FileReader();
     reader.onload = function (event) {
       resolve(event.target.result);
@@ -76,8 +103,9 @@ async function bufferToDataUrl(buf) {
     reader.readAsDataURL(blob);
   });
 }
+
 //open db and initialize it if necessary
-const openDB = (config, onOpened) => {
+function openDB(config, onOpened) {
   const { dbName, version, table, columns } = config;
   if (typeof window === 'undefined') {
     return;
@@ -102,19 +130,19 @@ const openDB = (config, onOpened) => {
 
   dbOpen.onsuccess = () => {
     const db = dbOpen.result;
-
+    // lock store for writing
-    const // lock store for writing
+    const writeTransaction = db.transaction([table], 'readwrite');
-      writeTransaction = db.transaction([table], 'readwrite'),
     // get object store
-      objectStore = writeTransaction.objectStore(table);
+    const objectStore = writeTransaction.objectStore(table);
     onOpened(objectStore, db);
   };
-};
+  return dbOpen;
+}
 
-const processFilesForIDB = async (files) => {
+async function processFilesForIDB(files) {
-  return await Promise.all(
+  return Promise.all(
     Array.from(files)
-      .map(async (s) => {
+      .map((s) => {
         const title = s.name;
 
         if (!isAudioFile(title)) {
@@ -123,34 +151,42 @@ const processFilesForIDB = async (files) => {
         //create obscured url to file system that can be fetched
         const sUrl = URL.createObjectURL(s);
         //fetch the sound and turn it into a buffer array
-        const buf = await fetch(sUrl).then((res) => res.arrayBuffer());
+        return fetch(sUrl).then((res) => {
-        //create a url base64 containing all of the buffer data
+          return res.blob().then((blob) => {
-        const base64 = await bufferToDataUrl(buf);
             const path = s.webkitRelativePath;
-        const id = path?.length ? path : title;
+            let id = path?.length ? path : title;
+            if (id == null || title == null || blob == null) {
+              return;
+            }
             return {
               title,
-          blob: base64,
+              blob,
               id,
             };
+          });
+        });
       })
       .filter(Boolean),
   ).catch((error) => {
     logger('Something went wrong while processing uploaded files', 'error');
     console.error(error);
   });
-};
+}
 
-export const uploadSamplesToDB = async (config, files) => {
+export async function uploadSamplesToDB(config, files) {
+  logger('procesing user samples...');
   await processFilesForIDB(files).then((files) => {
+    logger('user samples processed... opening db');
     const onOpened = (objectStore, _db) => {
+      logger('index db opened... writing files to db');
       files.forEach((file) => {
         if (file == null) {
           return;
         }
         objectStore.put(file);
       });
+      logger('user samples written successfully');
     };
     openDB(config, onOpened);
   });
-};
+}

website/src/repl/panel/ImportSoundsButton.jsx

@@ -33,7 +33,7 @@ export default function ImportSoundsButton({ onComplete }) {
         directory=""
         webkitdirectory=""
         multiple
-        accept="audio/*, .wav, .mp3, .m4a, .flac"
+        accept="audio/*, .wav, .mp3, .m4a, .flac, .aac, .ogg"
         onChange={() => {
           onChange();
         }}