From 890a1dd9a29ca8843d0360aa01f72e1d51cce68f Mon Sep 17 00:00:00 2001 From: Miraculous Owonubi Date: Sun, 4 Sep 2022 06:12:27 +0400 Subject: [PATCH] feat: replace native ffmpeg with bundled wasm version (#305) --- .eslintrc | 3 + .github/workflows/tests.yml | 42 ++-- Dockerfile | 2 +- README.md | 21 -- cli.js | 382 ++++++++++++++++++++---------------- package-lock.json | 133 +++++++------ package.json | 4 +- src/async_queue.js | 12 ++ src/file_mgr.js | 82 ++++---- src/symbols.js | 2 + test/index.js | 2 +- yarn.lock | 58 +++--- 12 files changed, 405 insertions(+), 338 deletions(-) diff --git a/.eslintrc b/.eslintrc index cd32e04..023d62c 100755 --- a/.eslintrc +++ b/.eslintrc @@ -4,6 +4,9 @@ "es6": true, "node": true }, + "globals": { + "globalThis": false + }, "parserOptions": { "sourceType": "module", "ecmaVersion": "latest", diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f956ab9..9fb15e2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,6 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] schedule: - cron: '0 0 * * WED,SAT' # 00:00 on Wednesdays and Saturdays, weekly. @@ -120,24 +119,31 @@ jobs: - name: Checkout Repository uses: actions/checkout@v3 + - name: Get Git SHAs + id: get-shas + if: github.event_name == 'pull_request' + run: | + BASE_SHA=$( echo ${{ github.event.pull_request.base.sha }} | head -c7 ) + echo "::set-output name=base_sha::$BASE_SHA" + HEAD_SHA=$( echo ${{ github.event.pull_request.head.sha }} | head -c7 ) + echo "::set-output name=head_sha::$HEAD_SHA" + - name: Extract metadata (tags, labels) for Docker - id: meta + id: docker-meta uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 with: images: freyrcli/freyrjs-git tags: | type=ref,event=pr type=ref,event=branch - type=sha,format=short,prefix= + type=raw,value=${{ steps.get-shas.outputs.head_sha }} - name: Extract Tags - id: meta2 + id: get-docker-tag if: github.event_name == 'pull_request' run: | - PR_TAG=$( echo "${{ steps.meta.outputs.tags }}" | sed 's/freyrcli\/freyrjs-git://g' ) + PR_TAG=$( echo "${{ steps.docker-meta.outputs.tags }}" | sed 's/freyrcli\/freyrjs-git://g' ) echo "::set-output name=tag::$PR_TAG" - SHA=$( echo ${{ github.event.pull_request.base.sha }} | head -c7 ) - echo "::set-output name=sha::$SHA" - name: Report Docker Image Build Status uses: marocchino/sticky-pull-request-comment@39c5b5dc7717447d0cba270cd115037d32d28443 @@ -155,10 +161,10 @@ jobs: **A docker image for this PR is being built!** ```console - docker pull freyrcli/freyrjs-git:${{steps.meta2.outputs.tag}} + docker pull freyrcli/freyrjs-git:${{ steps.get-docker-tag.outputs.tag }} ``` - | [**Base Branch (${{ github.event.pull_request.base.ref }})**][base-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{steps.meta2.outputs.sha}}?color=gray&label=%20&logo=docker)][base-url] | + | [**Base Branch (${{ github.event.pull_request.base.ref }})**][base-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{ steps.get-shas.outputs.base_sha }}?color=gray&label=%20&logo=docker)][base-url] | | :-: | - | --- @@ -173,7 +179,7 @@ jobs: - [base-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{steps.meta2.outputs.sha}} + [base-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{ steps.get-shas.outputs.base_sha }} - name: Set up QEMU uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 @@ -193,8 +199,8 @@ jobs: context: . push: true platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max @@ -212,12 +218,12 @@ jobs: **A docker image for this PR has been built!** ```console - docker pull freyrcli/freyrjs-git:${{steps.meta2.outputs.tag}} + docker pull freyrcli/freyrjs-git:${{ steps.get-docker-tag.outputs.tag }} ``` - | [**Base Branch (${{ github.event.pull_request.base.ref }})**][base-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{steps.meta2.outputs.sha}}?color=gray&label=%20&logo=docker)][base-url] | + | [**Base Branch (${{ github.event.pull_request.base.ref }})**][base-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{ steps.get-shas.outputs.base_sha }}?color=gray&label=%20&logo=docker)][base-url] | | :-: | - | - | [**This Patch**][pr-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{steps.meta2.outputs.tag}}?color=gray&label=%20&logo=docker)][pr-url] | + | [**This Patch**][pr-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{ steps.get-docker-tag.outputs.tag }}?color=gray&label=%20&logo=docker)][pr-url] | [![][compare-img]][compare-url] @@ -233,10 +239,10 @@ jobs: - [base-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{steps.meta2.outputs.sha}} - [pr-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{steps.meta2.outputs.tag}} + [base-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{ steps.get-shas.outputs.base_sha }} + [pr-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{ steps.get-docker-tag.outputs.tag }} [compare-img]: https://img.shields.io/badge/%20-compare-gray?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+CiAgIDxwYXRoIGZpbGw9IiNmZmZmZmYiIGQ9Ik0zLDFDMS44OSwxIDEsMS44OSAxLDNWMTRDMSwxNS4xMSAxLjg5LDE2IDMsMTZINVYxNEgzVjNIMTRWNUgxNlYzQzE2LDEuODkgMTUuMTEsMSAxNCwxSDNNOSw3QzcuODksNyA3LDcuODkgNyw5VjExSDlWOUgxMVY3SDlNMTMsN1Y5SDE0VjEwSDE2VjdIMTNNMTgsN1Y5SDIwVjIwSDlWMThIN1YyMEM3LDIxLjExIDcuODksMjIgOSwyMkgyMEMyMS4xMSwyMiAyMiwyMS4xMSAyMiwyMFY5QzIyLDcuODkgMjEuMTEsNyAyMCw3SDE4TTE0LDEyVjE0SDEyVjE2SDE0QzE1LjExLDE2IDE2LDE1LjExIDE2LDE0VjEySDE0TTcsMTNWMTZIMTBWMTRIOVYxM0g3WiIgLz4KPC9zdmc+ - [compare-url]: https://portal.slim.dev/home/diff/dockerhub%3A%2F%2Fdockerhub.public%2Ffreyrcli%2Ffreyrjs-git%3A${{steps.meta2.outputs.tag}}#file-system + [compare-url]: https://portal.slim.dev/home/diff/dockerhub%3A%2F%2Fdockerhub.public%2Ffreyrcli%2Ffreyrjs-git%3A${{ steps.get-docker-tag.outputs.tag }}#file-system linter: runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index 7e3011a..9922975 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN go install github.com/tj/node-prune@1159d4c \ FROM alpine:3.16.2 as base # hadolint ignore=DL3018 -RUN apk add --no-cache nodejs ffmpeg python3 \ +RUN apk add --no-cache nodejs python3 \ && find /usr/lib/python3* \ \( -type d -name __pycache__ -o -type f -name '*.whl' \) \ -exec rm -r {} \+ diff --git a/README.md b/README.md index f8fe392..2b9cd63 100644 --- a/README.md +++ b/README.md @@ -130,25 +130,6 @@ Here's a list of the metadata that freyr can extract from each streaming service -
- ffmpeg >= v0.9 - - - Download for your individual platforms here - - - - Windows + macOS: - - Ensure to extract the `ffmpeg` binary from the compressed file, if it's in one. - - make sure it's available in your `PATH` - - otherwise, set `FFMPEG_PATH` to explicitly specify binary to use - - Linux: _(check individual package managers)_ - - Debian: The `ppa:mc3man/trusty-media` PPA provides recent builds - - Arch Linux: `sudo pacman -S ffmpeg` - - Android (Termux): `apt install ffmpeg` - - Alpine Linux: `sudo apk add ffmpeg` - -
-
AtomicParsley >= (v0.9.6 | 20200701) @@ -315,7 +296,6 @@ Options: --no-auth skip authentication procedure --no-browser disable auto-launching of user browser --no-net-check disable internet connection check - --ffmpeg explicit path to the ffmpeg binary --atomic-parsley explicit path to the atomic-parsley binary --no-stats don't show the stats on completion --pulsate-bar show a pulsating bar @@ -325,7 +305,6 @@ Options: Environment Variables: SHOW_DEBUG_STACK show extended debug information - FFMPEG_PATH custom ffmpeg path, alternatively use `--ffmpeg` ATOMIC_PARSLEY_PATH custom AtomicParsley path, alternatively use `--atomic-parsley` Info: diff --git a/cli.js b/cli.js index 445e3e7..40626d4 100755 --- a/cli.js +++ b/cli.js @@ -9,7 +9,6 @@ import {promises as fs, constants as fs_constants, createReadStream, createWrite import Conf from 'conf'; import open from 'open'; import xget from 'libxget'; -import ffmpeg from 'fluent-ffmpeg'; import merge2 from 'merge2'; import mkdirp from 'mkdirp'; import xbytes from 'xbytes'; @@ -26,6 +25,7 @@ import {isBinaryFile} from 'isbinaryfile'; import {fileTypeFromFile} from 'file-type'; import {program as commander} from 'commander'; import {decode as entityDecode} from 'html-entities'; +import {createFFmpeg, fetchFile} from '@ffmpeg/ffmpeg'; import _merge from 'lodash.merge'; import _mergeWith from 'lodash.mergewith'; @@ -633,13 +633,6 @@ async function init(packageJson, queries, options) { let atomicParsley; try { - if (options.ffmpeg) { - if (!(await maybeStat(options.ffmpeg))) throw new Error(`\x1b[31mffmpeg\x1b[0m: Binary not found [${options.ffmpeg}]`); - if (!(await isBinaryFile(options.ffmpeg))) - stackLogger.warn('\x1b[33mffmpeg\x1b[0m: Detected non-binary file, trying anyways...'); - ffmpeg.setFfmpegPath(options.ffmpeg); - } - if (options.atomicParsley) { if (!(await maybeStat(options.atomicParsley))) throw new Error(`\x1b[31mAtomicParsley\x1b[0m: Binary not found [${options.atomicParsley}]`); @@ -654,7 +647,7 @@ async function init(packageJson, queries, options) { async function createPlaylist(header, stats, logger, filename, playlistTitle, shouldAppend) { if (options.playlist !== false) { - const validStats = stats.filter(stat => (stat.code === 0 ? stat.complete : !stat.code)); + const validStats = stats.filter(stat => (stat[symbols.errorCode] === 0 ? stat.complete : !stat[symbols.errorCode])); if (validStats.length) { logger.print('[\u2022] Creating playlist...'); const playlistFile = xpath.join( @@ -704,9 +697,8 @@ async function init(packageJson, queries, options) { function downloadToStream({urlOrFragments, outputFile, logger, opts}) { opts = {tag: '', successMessage: '', failureMessage: '', retryMessage: '', ...opts}; - [opts.tag, opts.errorHandler, opts.retryMessage, opts.failureMessage, opts.successMessage, opts.altMessage] = [ + [opts.tag, opts.retryMessage, opts.failureMessage, opts.successMessage, opts.altMessage] = [ opts.tag, - opts.errorHandler, opts.retryMessage, opts.failureMessage, opts.successMessage, @@ -748,7 +740,6 @@ async function init(packageJson, queries, options) { if (!options.bar) logger.write('\x1b[G\x1b[K'); logger.write(opts.failureMessage(err), '\n'); } - opts.errorHandler(err); rej(err); }); @@ -822,7 +813,6 @@ async function init(packageJson, queries, options) { logger.write('\x1b[G\x1b[K'); logger.write(opts.failureMessage(err), '\n'); } - opts.errorHandler(err); rej(err); }); return !options.bar ? feed : feed.pipe(barGen.next(frag.size)); @@ -848,70 +838,88 @@ async function init(packageJson, queries, options) { Config.concurrency.downloader, async ({track, meta, feedMeta, trackLogger}) => { const baseCacheDir = 'fr3yrcach3'; + const imageFile = await fileMgr({ filename: `freyrcli-${meta.fingerprint}.x4i`, tempdir: Config.dirs.cacheDir === '' ? undefined : Config.dirs.cacheDir, dirname: baseCacheDir, keep: true, }); - const imageBytesWritten = await downloadToStream({ - urlOrFragments: track.getImage(Config.image.width, Config.image.height), - outputFile: imageFile.path, - logger: trackLogger, - opts: { - tag: '[Retrieving album art]...', - errorHandler: () => imageFile.removeCallback(), - retryMessage: data => trackLogger.getText(`| ${getRetryMessage(data)}`), - resumeHandler: offset => trackLogger.log(cStringd(`| :{color(yellow)}{i}:{color:close(yellow)} Resuming at ${offset}`)), - failureMessage: err => - trackLogger.getText(`| [\u2715] Failed to get album art${err ? ` [${err.code || err.message}]` : ''}`), - successMessage: trackLogger.getText(`| [\u2713] Got album art`), - altMessage: trackLogger.getText('| \u27a4 Downloading album art...'), - }, - }).catch(err => Promise.reject({err, code: 3})); + + let imageBytesWritten = 0; + try { + imageBytesWritten = await downloadToStream({ + urlOrFragments: track.getImage(Config.image.width, Config.image.height), + outputFile: imageFile.path, + logger: trackLogger, + opts: { + tag: '[Retrieving album art]...', + retryMessage: data => trackLogger.getText(`| ${getRetryMessage(data)}`), + resumeHandler: offset => + trackLogger.log(cStringd(`| :{color(yellow)}{i}:{color:close(yellow)} Resuming at ${offset}`)), + failureMessage: err => + trackLogger.getText(`| [\u2715] Failed to get album art${err ? ` [${err.code || err.message}]` : ''}`), + successMessage: trackLogger.getText(`| [\u2713] Got album art`), + altMessage: trackLogger.getText('| \u27a4 Downloading album art...'), + }, + }); + } catch (err) { + await imageFile.removeCallback(); + throw {err, [symbols.errorCode]: 3}; + } + const rawAudio = await fileMgr({ filename: `freyrcli-${meta.fingerprint}.x4a`, tempdir: Config.dirs.cacheDir === '' ? undefined : Config.dirs.cacheDir, dirname: baseCacheDir, keep: true, }); - const audioBytesWritten = await downloadToStream( - _merge( - { - outputFile: rawAudio.path, - logger: trackLogger, - opts: { - tag: `[‘${meta.trackName}’]`, - retryMessage: data => trackLogger.getText(`| ${getRetryMessage(data)}`), - resumeHandler: offset => - trackLogger.log(cStringd(`| :{color(yellow)}{i}:{color:close(yellow)} Resuming at ${offset}`)), - successMessage: trackLogger.getText('| [\u2713] Got raw track file'), - altMessage: trackLogger.getText('| \u27a4 Downloading track...'), - }, - }, - feedMeta.protocol !== 'http_dash_segments' - ? { - urlOrFragments: feedMeta.url, - opts: { - errorHandler: () => rawAudio.removeCallback(), - failureMessage: err => - trackLogger.getText(`| [\u2715] Failed to get raw media stream${err ? ` [${err.code || err.message}]` : ''}`), - }, - } - : { - urlOrFragments: feedMeta.fragments.map(({path}) => ({ - url: `${feedMeta.fragment_base_url}${path}`, - ...(([, min, max]) => ({min: +min, max: +max, size: +max - +min + 1}))(path.match(/range\/(\d+)-(\d+)$/)), - })), - opts: { - failureMessage: err => - trackLogger.getText( - `| [\u2715] Segment error while getting raw media${err ? ` [${err.code || err.message}]` : ''}`, - ), - }, + + let audioBytesWritten = 0; + try { + audioBytesWritten = await downloadToStream( + _merge( + { + outputFile: rawAudio.path, + logger: trackLogger, + opts: { + tag: `[‘${meta.trackName}’]`, + retryMessage: data => trackLogger.getText(`| ${getRetryMessage(data)}`), + resumeHandler: offset => + trackLogger.log(cStringd(`| :{color(yellow)}{i}:{color:close(yellow)} Resuming at ${offset}`)), + successMessage: trackLogger.getText('| [\u2713] Got raw track file'), + altMessage: trackLogger.getText('| \u27a4 Downloading track...'), }, - ), - ).catch(err => Promise.reject({err, code: 4})); + }, + feedMeta.protocol !== 'http_dash_segments' + ? { + urlOrFragments: feedMeta.url, + opts: { + failureMessage: err => + trackLogger.getText( + `| [\u2715] Failed to get raw media stream${err ? ` [${err.code || err.message}]` : ''}`, + ), + }, + } + : { + urlOrFragments: feedMeta.fragments.map(({path}) => ({ + url: `${feedMeta.fragment_base_url}${path}`, + ...(([, min, max]) => ({min: +min, max: +max, size: +max - +min + 1}))(path.match(/range\/(\d+)-(\d+)$/)), + })), + opts: { + failureMessage: err => + trackLogger.getText( + `| [\u2715] Segment error while getting raw media${err ? ` [${err.code || err.message}]` : ''}`, + ), + }, + }, + ), + ); + } catch (err) { + await rawAudio.removeCallback(); + throw {err, [symbols.errorCode]: 4}; + } + return { image: {file: imageFile, bytesWritten: imageBytesWritten}, audio: {file: rawAudio, bytesWritten: audioBytesWritten}, @@ -923,94 +931,124 @@ async function init(packageJson, queries, options) { 'cli:postprocessor:embedQueue', Config.concurrency.embedder, async ({track, meta, files, audioSource}) => { - return Promise.promisify(atomicParsley)(meta.outFilePath, { - overWrite: '', // overwrite the file + try { + await Promise.promisify(atomicParsley)(meta.outFilePath, { + overWrite: '', // overwrite the file - title: track.name, // ©nam - artist: track.artists[0], // ©ART - composer: track.composers, // ©wrt - album: track.album, // ©alb - genre: (genre => (genre ? genre.concat(' ') : ''))((track.genres || [])[0]), // ©gen | gnre - tracknum: `${track.track_number}/${track.total_tracks}`, // trkn - disk: `${track.disc_number}/${track.disc_number}`, // disk - year: new Date(track.release_date).toISOString().split('T')[0], // ©day - compilation: track.compilation, // ©cpil - gapless: options.gapless, // pgap - rDNSatom: [ - // ---- - ['Digital Media', 'name=MEDIA', 'domain=com.apple.iTunes'], - [track.isrc, 'name=ISRC', 'domain=com.apple.iTunes'], - [track.artists[0], 'name=ARTISTS', 'domain=com.apple.iTunes'], - [track.label, 'name=LABEL', 'domain=com.apple.iTunes'], - [`${meta.service[symbols.meta].DESC}: ${track.uri}`, 'name=SOURCE', 'domain=com.apple.iTunes'], - [ - `${audioSource.service[symbols.meta].DESC}: ${audioSource.source.videoId}`, - 'name=PROVIDER', - 'domain=com.apple.iTunes', + title: track.name, // ©nam + artist: track.artists[0], // ©ART + composer: track.composers, // ©wrt + album: track.album, // ©alb + genre: (genre => (genre ? genre.concat(' ') : ''))((track.genres || [])[0]), // ©gen | gnre + tracknum: `${track.track_number}/${track.total_tracks}`, // trkn + disk: `${track.disc_number}/${track.disc_number}`, // disk + year: new Date(track.release_date).toISOString().split('T')[0], // ©day + compilation: track.compilation, // ©cpil + gapless: options.gapless, // pgap + rDNSatom: [ + // ---- + ['Digital Media', 'name=MEDIA', 'domain=com.apple.iTunes'], + [track.isrc, 'name=ISRC', 'domain=com.apple.iTunes'], + [track.artists[0], 'name=ARTISTS', 'domain=com.apple.iTunes'], + [track.label, 'name=LABEL', 'domain=com.apple.iTunes'], + [`${meta.service[symbols.meta].DESC}: ${track.uri}`, 'name=SOURCE', 'domain=com.apple.iTunes'], + [ + `${audioSource.service[symbols.meta].DESC}: ${audioSource.source.videoId}`, + 'name=PROVIDER', + 'domain=com.apple.iTunes', + ], ], - ], - advisory: ['explicit', 'clean'].includes(track.contentRating) // rtng - ? track.contentRating - : track.contentRating === true - ? 'explicit' - : 'Inoffensive', - stik: 'Normal', // stik - // geID: 0, // geID: genreID. See `AtomicParsley --genre-list` - // sfID: 0, // ~~~~: store front ID - // cnID: 0, // cnID: catalog ID - albumArtist: track.album_artist, // aART - // ownr? - purchaseDate: 'timestamp', // purd - apID: 'cli@freyr.git', // apID - copyright: track.copyrights.sort(({type}) => (type === 'P' ? -1 : 1))[0].text, // cprt - encodingTool: `freyr-js cli v${packageJson.version}`, // ©too - encodedBy: 'd3vc0dr', // ©enc - artwork: files.image.file.path, // covr - // sortOrder: [ - // ['name', 'NAME'], // sonm - // ['album', 'NAME'], // soal - // ['artist', 'NAME'], // soar - // ['albumartist', 'NAME'], // soaa - // ], - }) - .finally(() => files.image.file.removeCallback()) - .catch(err => Promise.reject({err, code: 8})); + advisory: ['explicit', 'clean'].includes(track.contentRating) // rtng + ? track.contentRating + : track.contentRating === true + ? 'explicit' + : 'Inoffensive', + stik: 'Normal', // stik + // geID: 0, // geID: genreID. See `AtomicParsley --genre-list` + // sfID: 0, // ~~~~: store front ID + // cnID: 0, // cnID: catalog ID + albumArtist: track.album_artist, // aART + // ownr? + purchaseDate: 'timestamp', // purd + apID: 'cli@freyr.git', // apID + copyright: track.copyrights.sort(({type}) => (type === 'P' ? -1 : 1))[0].text, // cprt + encodingTool: `freyr-js cli v${packageJson.version}`, // ©too + encodedBy: 'd3vc0dr', // ©enc + artwork: files.image.file.path, // covr + // sortOrder: [ + // ['name', 'NAME'], // sonm + // ['album', 'NAME'], // soal + // ['artist', 'NAME'], // soar + // ['albumartist', 'NAME'], // soaa + // ], + }); + } catch (err) { + throw {err, [symbols.errorCode]: 8}; + } finally { + await files.image.file.removeCallback(); + } }, ); + delete globalThis.fetch; + const encodeQueue = new AsyncQueue( 'cli:postprocessor:encodeQueue', Config.concurrency.encoder, - async ({track, meta, files}) => { - return new Promise((res, rej) => - ffmpeg() - .addInput(files.audio.file.path) - .audioCodec('aac') - .audioBitrate(options.bitrate) - .audioFrequency(44100) - .noVideo() - .setDuration(TimeFormat.fromMs(track.duration, 'hh:mm:ss.sss')) - .toFormat('ipod') - .saveToFile(meta.outFilePath) - .on('error', err => rej({err, code: 7})) - .on('end', res), - ).finally(() => files.audio.file.removeCallback()); - }, + AsyncQueue.provision( + async () => { + let ffmpeg = createFFmpeg({log: false}); + await ffmpeg.load(); + return ffmpeg; + }, + async (ffmpeg, {track, meta, files}) => { + let infile = xpath.basename(files.audio.file.path); + let outfile = xpath.basename(files.audio.file.path.replace(/\.x4a$/, '.m4a')); + try { + ffmpeg.FS('writeFile', infile, await fetchFile(files.audio.file.path)); + await ffmpeg.run( + '-i', + infile, + '-acodec', + 'aac', + '-b:a', + options.bitrate, + '-ar', + '44100', + '-vn', + '-t', + TimeFormat.fromMs(track.duration, 'hh:mm:ss.sss'), + '-f', + 'ipod', + outfile, + ); + await fs.writeFile(meta.outFilePath, ffmpeg.FS('readFile', outfile)); + } catch (err) { + throw {err, [symbols.errorCode]: 7}; + } finally { + await files.audio.file.removeCallback(); + } + }, + ), ); - const postProcessor = new AsyncQueue('cli:postProcessor', 4, async ({track, meta, files, audioSource}) => { - await mkdirp(meta.outFileDir).catch(err => Promise.reject({err, code: 6})); - const wroteImage = - !!options.cover && - (await (async outArtPath => - (await maybeStat(outArtPath).then(stat => stat && stat.isFile())) || - (await fs.copyFile(files.image.file.path, outArtPath), true))( - xpath.join(meta.outFileDir, `${options.cover}.${(await fileTypeFromFile(files.image.file.path)).ext}`), - )); - await encodeQueue.push({track, meta, files}); - await embedQueue.push({track, meta, files, audioSource}); - return {wroteImage, finalSize: (await fs.stat(meta.outFilePath)).size}; - }); + const postProcessor = new AsyncQueue( + 'cli:postProcessor', + Math.max(Config.concurrency.encoder, Config.concurrency.embedder), + async ({track, meta, files, audioSource}) => { + await mkdirp(meta.outFileDir).catch(err => Promise.reject({err, [symbols.errorCode]: 6})); + const wroteImage = + !!options.cover && + (await (async outArtPath => + (await maybeStat(outArtPath).then(stat => stat && stat.isFile())) || + (await fs.copyFile(files.image.file.path, outArtPath), true))( + xpath.join(meta.outFileDir, `${options.cover}.${(await fileTypeFromFile(files.image.file.path)).ext}`), + )); + await encodeQueue.push({track, meta, files}); + await embedQueue.push({track, meta, files, audioSource}); + return {wroteImage, finalSize: (await fs.stat(meta.outFilePath)).size}; + }, + ); function buildSourceCollectorFor(track, selector) { async function handleSource(iterator, lastErr) { @@ -1065,13 +1103,13 @@ async function init(packageJson, queries, options) { const filterStat = options.filter(track, false); if (!filterStat.status) { trackLogger.log("| [\u2022] Didn't match filter. Skipping..."); - return {meta, code: 0, skip_reason: `filtered out: ${filterStat.reason.message}`, complete: false}; + return {meta, [symbols.errorCode]: 0, skip_reason: `filtered out: ${filterStat.reason.message}`, complete: false}; } if (props.fileExists) { if (!props.processTrack) { trackLogger.log('| [\u00bb] Track exists. Skipping...'); - return {meta, code: 0, skip_reason: 'exists', complete: true}; + return {meta, [symbols.errorCode]: 0, skip_reason: 'exists', complete: true}; } trackLogger.log('| [\u2022] Track exists. Overwriting...'); } @@ -1083,12 +1121,12 @@ async function init(packageJson, queries, options) { onPass: ({sources}) => `[success, found ${sources.length} source${sources.length === 1 ? '' : 's'}]\n`, }), ); - if ('err' in audioSource) return {meta, code: 1, err: audioSource.err}; // zero sources found + if ('err' in audioSource) return {meta, [symbols.errorCode]: 1, err: audioSource.err}; // zero sources found const audioFeeds = await processPromise(audioSource.feeds, trackLogger, { onInit: '| \u27a4 Awaiting audiofeeds...', noVal: () => '[Unable to collect source feeds]\n', }); - if (!audioFeeds || audioFeeds.err) return {meta, err: (audioFeeds || {}).err, code: 2}; + if (!audioFeeds || audioFeeds.err) return {meta, err: (audioFeeds || {}).err, [symbols.errorCode]: 2}; const [feedMeta] = audioFeeds.formats .filter(meta => 'abr' in meta && !('vbr' in meta)) @@ -1097,11 +1135,15 @@ async function init(packageJson, queries, options) { meta.fingerprint = crypto.createHash('md5').update(`${audioSource.source.videoId} ${feedMeta.format_id}`).digest('hex'); const files = await downloadQueue .push({track, meta, feedMeta, trackLogger}) - .catch(errObject => Promise.reject({meta, code: 5, ...(errObject.code ? errObject : {err: errObject})})); + .catch(errObject => + Promise.reject({meta, [symbols.errorCode]: 5, ...(symbols.errorCode in errObject ? errObject : {err: errObject})}), + ); trackLogger.log(`| [\u2022] Post Processing...`); return { files, - postprocess: postProcessor.push({track, meta, files, audioSource}).catch(errObject => ({code: 9, ...errObject})), + postprocess: postProcessor + .push({track, meta, files, audioSource}) + .catch(errObject => ({[symbols.errorCode]: 9, ...(symbols.errorCode in errObject ? errObject : {err: errObject})})), }; }); @@ -1112,11 +1154,11 @@ async function init(packageJson, queries, options) { try { if (!(track = await track)) throw new Error('no data recieved from track'); } catch (err) { - return {code: -1, err}; + return {[symbols.errorCode]: -1, err}; } if ((track[symbols.errorStack] || {}).code === 1) return { - code: -1, + [symbols.errorCode]: -1, err: new Error("local-typed tracks aren't supported"), meta: {track: {uri: track[symbols.errorStack].uri}}, }; @@ -1140,7 +1182,7 @@ async function init(packageJson, queries, options) { return trackQueue .push({track, meta, props: {collectSources, fileExists, processTrack, logger}}) .then(trackObject => ({...trackObject, meta})) - .catch(errObject => ({meta, code: 10, ...errObject})); + .catch(errObject => ({meta, [symbols.errorCode]: 10, ...errObject})); }, ); @@ -1326,34 +1368,34 @@ async function init(packageJson, queries, options) { await Promise.mapSeries(trackStats, async trackStat => { if (trackStat.postprocess) { trackStat.postprocess = await trackStat.postprocess; - if ('code' in trackStat.postprocess) { - trackStat.code = trackStat.postprocess.code; + if (symbols.errorCode in trackStat.postprocess) { + trackStat[symbols.errorCode] = trackStat.postprocess[symbols.errorCode]; trackStat.err = trackStat.postprocess.err; } } - if (trackStat.code) { + if (trackStat[symbols.errorCode]) { const reason = - trackStat.code === -1 + trackStat[symbols.errorCode] === -1 ? 'Failed getting track data' - : trackStat.code === 1 + : trackStat[symbols.errorCode] === 1 ? 'Failed collecting sources' - : trackStat.code === 2 + : trackStat[symbols.errorCode] === 2 ? 'Error while collecting sources feeds' - : trackStat.code === 3 + : trackStat[symbols.errorCode] === 3 ? 'Error downloading album art' - : trackStat.code === 4 + : trackStat[symbols.errorCode] === 4 ? 'Error downloading raw audio' - : trackStat.code === 5 + : trackStat[symbols.errorCode] === 5 ? 'Unknown Download Error' - : trackStat.code === 6 + : trackStat[symbols.errorCode] === 6 ? 'Error ensuring directory integrity' - : trackStat.code === 7 + : trackStat[symbols.errorCode] === 7 ? 'Error while encoding audio' - : trackStat.code === 8 + : trackStat[symbols.errorCode] === 8 ? 'Failed while embedding metadata' - : trackStat.code === 9 - ? 'Unknown postprocessing error' - : 'Unknown track processing error'; + : trackStat[symbols.errorCode] === 9 + ? 'Unexpected postprocessing error' + : 'Unexpected track processing error'; embedLogger.error( `\u2022 [\u2715] ${trackStat.meta && trackStat.meta.trackName ? `${trackStat.meta.trackName}` : ''}${ trackStat.meta && trackStat.meta.track.uri ? ` [${trackStat.meta.track.uri}]` : '' @@ -1361,7 +1403,7 @@ async function init(packageJson, queries, options) { trackStat.err ? ` [${trackStat.err['SHOW_DEBUG_STACK' in process.env ? 'stack' : 'message'] || trackStat.err}]` : '' })`, ); - } else if (trackStat.code === 0) + } else if (trackStat[symbols.errorCode] === 0) embedLogger.log(`\u2022 [\u00bb] ${trackStat.meta.trackName} (skipped: ${trackStat.skip_reason})`); else embedLogger.log( @@ -1408,10 +1450,10 @@ async function init(packageJson, queries, options) { total.mediaSize += audio; total.imageSize += image; } - if (current.code === 0) + if (current[symbols.errorCode] === 0) if (current.complete) total.passed += 1; else total.skipped += 1; - else if (!('code' in current)) (total.new += 1), (total.passed += 1); + else if (!(symbols.errorCode in current)) (total.new += 1), (total.passed += 1); else total.failed += 1; return total; }, @@ -1554,7 +1596,6 @@ function prepCli(packageJson) { .option('--no-browser', 'disable auto-launching of user browser') .option('--no-net-check', 'disable internet connection check') .option('--no-bar', 'disable the progress bar') - .option('--ffmpeg ', 'explicit path to the ffmpeg binary') .option('--atomic-parsley ', 'explicit path to the atomic-parsley binary') .option('--no-stats', "don't show the stats on completion") .option('--pulsate-bar', 'show a pulsating bar') @@ -1570,7 +1611,6 @@ function prepCli(packageJson) { console.log(''); console.log('Environment Variables:'); console.log(' SHOW_DEBUG_STACK show extended debug information'); - console.log(' FFMPEG_PATH custom ffmpeg path, alternatively use `--ffmpeg`'); console.log(' ATOMIC_PARSLEY_PATH custom AtomicParsley path, alternatively use `--atomic-parsley`'); console.log(''); console.log('Info:'); diff --git a/package-lock.json b/package-lock.json index 8992cbc..998a3cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,8 @@ "version": "0.8.1", "license": "Apache-2.0", "dependencies": { + "@ffmpeg/core": "^0.10.0", + "@ffmpeg/ffmpeg": "^0.10.1", "@yujinakayama/apple-music": "^0.4.1", "async": "^3.2.4", "bluebird": "^3.7.2", @@ -21,7 +23,6 @@ "express": "^4.18.1", "file-type": "^18.0.0", "filenamify": "^5.1.1", - "fluent-ffmpeg": "^2.1.2", "got": "^12.1.0", "hh-mm-ss": "^1.2.0", "html-entities": "^2.3.3", @@ -42,7 +43,6 @@ "stringd": "^2.2.0", "stringd-colors": "^1.10.0", "stripchar": "^1.2.1", - "tmp": "^0.2.1", "xbytes": "^1.8.0", "xprogress": "^0.19.1", "youtube-dl-exec": "^2.1.4", @@ -118,6 +118,25 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, + "node_modules/@ffmpeg/core": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@ffmpeg/core/-/core-0.10.0.tgz", + "integrity": "sha512-qunWJl5PezpXEm31tb8Qu5z37B5KVA1VYZCpXchMhuAb3X9T7PuE3SlhOwphEoRhzaOa3lpofDfzihAUMFaVPQ==" + }, + "node_modules/@ffmpeg/ffmpeg": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.10.1.tgz", + "integrity": "sha512-ChQkH7Rh57hmVo1LhfQFibWX/xqneolJKSwItwZdKPcLZuKigtYAYDIvB55pDfP17VtR1R77SxgkB2/UApB+Og==", + "dependencies": { + "is-url": "^1.2.4", + "node-fetch": "^2.6.1", + "regenerator-runtime": "^0.13.7", + "resolve-url": "^0.2.1" + }, + "engines": { + "node": ">=12.16.1" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.10.4", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.4.tgz", @@ -2010,18 +2029,6 @@ "integrity": "sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==", "dev": true }, - "node_modules/fluent-ffmpeg": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz", - "integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=", - "dependencies": { - "async": ">=0.2.9", - "which": "^1.1.1" - }, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/follow-redirects": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.0.tgz", @@ -2086,7 +2093,8 @@ "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true }, "node_modules/function-bind": { "version": "1.1.1", @@ -2130,6 +2138,7 @@ "version": "7.1.7", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "dev": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -2628,6 +2637,11 @@ "node": ">= 12" } }, + "node_modules/is-url": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", + "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==" + }, "node_modules/is-wsl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", @@ -3612,6 +3626,11 @@ "node": "*" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.9", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" + }, "node_modules/regexpp": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", @@ -3646,6 +3665,12 @@ "node": ">=4" } }, + "node_modules/resolve-url": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", + "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==", + "deprecated": "https://github.com/lydell/resolve-url#deprecated" + }, "node_modules/responselike": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz", @@ -4081,17 +4106,6 @@ "next-tick": "1" } }, - "node_modules/tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dependencies": { - "rimraf": "^3.0.0" - }, - "engines": { - "node": ">=8.17.0" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -4274,17 +4288,6 @@ "webidl-conversions": "^3.0.0" } }, - "node_modules/which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "which": "bin/which" - } - }, "node_modules/window-size": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/window-size/-/window-size-1.1.1.tgz", @@ -4487,6 +4490,22 @@ } } }, + "@ffmpeg/core": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@ffmpeg/core/-/core-0.10.0.tgz", + "integrity": "sha512-qunWJl5PezpXEm31tb8Qu5z37B5KVA1VYZCpXchMhuAb3X9T7PuE3SlhOwphEoRhzaOa3lpofDfzihAUMFaVPQ==" + }, + "@ffmpeg/ffmpeg": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.10.1.tgz", + "integrity": "sha512-ChQkH7Rh57hmVo1LhfQFibWX/xqneolJKSwItwZdKPcLZuKigtYAYDIvB55pDfP17VtR1R77SxgkB2/UApB+Og==", + "requires": { + "is-url": "^1.2.4", + "node-fetch": "^2.6.1", + "regenerator-runtime": "^0.13.7", + "resolve-url": "^0.2.1" + } + }, "@humanwhocodes/config-array": { "version": "0.10.4", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.4.tgz", @@ -5924,15 +5943,6 @@ "integrity": "sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==", "dev": true }, - "fluent-ffmpeg": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz", - "integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=", - "requires": { - "async": ">=0.2.9", - "which": "^1.1.1" - } - }, "follow-redirects": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.0.tgz", @@ -6345,6 +6355,11 @@ "resolved": "https://registry.npmjs.org/is-unix/-/is-unix-2.0.1.tgz", "integrity": "sha512-RyKp5JtlRnfOvnKtfBMPLw9ocqDe1NlPQ8Bt+geVzKGfMnLGj8z/Y2HOmk/aMf47P4EbrEt9dN6YGTT1fx4mZA==" }, + "is-url": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", + "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==" + }, "is-wsl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", @@ -7055,6 +7070,11 @@ } } }, + "regenerator-runtime": { + "version": "0.13.9", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" + }, "regexpp": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", @@ -7077,6 +7097,11 @@ "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true }, + "resolve-url": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", + "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==" + }, "responselike": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz", @@ -7398,14 +7423,6 @@ "next-tick": "1" } }, - "tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "requires": { - "rimraf": "^3.0.0" - } - }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -7543,14 +7560,6 @@ "webidl-conversions": "^3.0.0" } }, - "which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "requires": { - "isexe": "^2.0.0" - } - }, "window-size": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/window-size/-/window-size-1.1.1.tgz", diff --git a/package.json b/package.json index ff29e78..91d076e 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,8 @@ }, "license": "Apache-2.0", "dependencies": { + "@ffmpeg/core": "^0.10.0", + "@ffmpeg/ffmpeg": "^0.10.1", "@yujinakayama/apple-music": "^0.4.1", "async": "^3.2.4", "bluebird": "^3.7.2", @@ -63,7 +65,6 @@ "express": "^4.18.1", "file-type": "^18.0.0", "filenamify": "^5.1.1", - "fluent-ffmpeg": "^2.1.2", "got": "^12.1.0", "hh-mm-ss": "^1.2.0", "html-entities": "^2.3.3", @@ -84,7 +85,6 @@ "stringd": "^2.2.0", "stringd-colors": "^1.10.0", "stripchar": "^1.2.1", - "tmp": "^0.2.1", "xbytes": "^1.8.0", "xprogress": "^0.19.1", "youtube-dl-exec": "^2.1.4", diff --git a/src/async_queue.js b/src/async_queue.js index 8e5253d..3904eac 100644 --- a/src/async_queue.js +++ b/src/async_queue.js @@ -65,6 +65,18 @@ export default class AsyncQueue { }, concurrency || 1); } + static provision(genFn, worker) { + let resources = []; + return async (...args) => { + let resource = resources.shift() || (await genFn()); + try { + return await worker(resource, ...args); + } finally { + resources.push(resource); + } + }; + } + #_register = (objects, meta, handler) => { const promises = (Array.isArray(objects) ? objects : [[objects, meta]]).map(objectBlocks => { const [data, args] = Array.isArray(objectBlocks) ? objectBlocks : [objectBlocks, meta]; diff --git a/src/file_mgr.js b/src/file_mgr.js index 0565022..1e0794f 100644 --- a/src/file_mgr.js +++ b/src/file_mgr.js @@ -1,11 +1,12 @@ -import {join} from 'path'; +import {join, resolve, dirname} from 'path'; import {tmpdir} from 'os'; +import {createHash} from 'crypto'; import {promises as fs, constants as fs_constants} from 'fs'; -import tmp from 'tmp'; import mkdirp from 'mkdirp'; import esMain from 'es-main'; +const openfiles = {}; const removeCallbacks = []; function garbageCollector() { @@ -23,49 +24,56 @@ function hookupListeners() { export default async function genFile(opts) { opts = opts || {}; - if (opts.filename) { - opts.tmpdir = opts.tmpdir || tmpdir(); - const dir = join(opts.tmpdir, opts.dirname || '.'); - await mkdirp(dir); - const path = join(dir, opts.filename); - const fd = await fs.open(path, fs_constants.O_CREAT | opts.mode); - hookupListeners(); - let closed = false; - const garbageHandler = async keep => { - if (closed) return; - await fd.close(); - closed = true; - if (!keep) await fs.unlink(path); - }; - removeCallbacks.push(garbageHandler.bind(opts.keep)); - return { - fd, + let mode = fs_constants.O_CREAT | opts.mode; + const path = resolve(join('tmpdir' in opts ? opts['tmpdir'] : tmpdir(), opts.dirname || '.', opts.filename)); + let id = createHash('md5').update(`Ξ${mode}${path}`).digest('hex'); + let file = openfiles[id]; + if (!file) { + await mkdirp(dirname(path)); + file = openfiles[id] = { path, - removeCallback: async () => { - await garbageHandler(false); - removeCallbacks.splice(removeCallbacks.indexOf(garbageHandler), 1); - }, + handle: await fs.open(path, mode), + refs: 1, + closed: false, + keep: false, }; - } - return new Promise((res, rej) => - tmp.file(opts, (err, name, fd, removeCallback) => (err ? rej(err) : res({fd, name, removeCallback}))), - ); + } else file.refs += 1; + hookupListeners(); + const garbageHandler = async keep => { + file.keep ||= keep !== undefined ? keep : opts.keep; + if ((file.refs = Math.max(0, file.refs - 1))) return; + if (file.closed) return; + file.closed = true; + delete openfiles[id]; + await file.handle.close(); + if (!file.keep) await fs.unlink(path); + }; + removeCallbacks.push(garbageHandler); + return { + path, + handle: file.handle, + removeCallback: async () => { + await garbageHandler(false); + removeCallbacks.splice(removeCallbacks.indexOf(garbageHandler), 1); + }, + }; } async function test() { const filename = 'freyr_mgr_temp_file'; - async function testMgr() { - const file = await genFile({filename}); + async function testMgr(args) { + const file = await genFile({filename, ...args}); console.log('mgr>', file); - file.removeCallback(); + return file; } - async function testTmp() { - const file = await genFile({name: filename}); - console.log('tmp>', file); - file.removeCallback(); - } - await testMgr(); - await testTmp(); + let a = await testMgr(); + let b = await testMgr(); + await testMgr({keep: true}); + let d = await testMgr(); + a.removeCallback(); + b.removeCallback(); + // c.removeCallback(); // calling this would negate the keep directive + d.removeCallback(); } if (esMain(import.meta)) test().catch(err => console.error('cli>', err)); diff --git a/src/symbols.js b/src/symbols.js index f7143cc..bbf7c91 100644 --- a/src/symbols.js +++ b/src/symbols.js @@ -1,7 +1,9 @@ const meta = Symbol('FreyrServiceMeta'); +const errorCode = Symbol('FreyrErrorCode'); const errorStack = Symbol('FreyrErrorStack'); export default { meta, + errorCode, errorStack, }; diff --git a/test/index.js b/test/index.js index c98db72..0361f45 100644 --- a/test/index.js +++ b/test/index.js @@ -117,7 +117,7 @@ async function run_tests(suite, args, i) { mode: fs_constants.W_OK, }); - logFile.stream = createWriteStream(null, {fd: logFile.fd}); + logFile.stream = createWriteStream(null, {fd: logFile.handle}); let logline = line => `│ ${line}`; diff --git a/yarn.lock b/yarn.lock index 11ada12..912ff53 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17,6 +17,21 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" +"@ffmpeg/core@^0.10.0": + version "0.10.0" + resolved "https://registry.npmjs.org/@ffmpeg/core/-/core-0.10.0.tgz" + integrity sha512-qunWJl5PezpXEm31tb8Qu5z37B5KVA1VYZCpXchMhuAb3X9T7PuE3SlhOwphEoRhzaOa3lpofDfzihAUMFaVPQ== + +"@ffmpeg/ffmpeg@^0.10.1": + version "0.10.1" + resolved "https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.10.1.tgz" + integrity sha512-ChQkH7Rh57hmVo1LhfQFibWX/xqneolJKSwItwZdKPcLZuKigtYAYDIvB55pDfP17VtR1R77SxgkB2/UApB+Og== + dependencies: + is-url "^1.2.4" + node-fetch "^2.6.1" + regenerator-runtime "^0.13.7" + resolve-url "^0.2.1" + "@humanwhocodes/config-array@^0.10.4": version "0.10.4" resolved "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.4.tgz" @@ -294,7 +309,7 @@ async.util.restparam@0.5.2: resolved "https://registry.npmjs.org/async.util.restparam/-/async.util.restparam-0.5.2.tgz" integrity sha1-A+/r88Ane5ciDlJaunUPXgT8gM0= -async@>=0.2.9, async@^3.2.4: +async@^3.2.4: version "3.2.4" resolved "https://registry.npmjs.org/async/-/async-3.2.4.tgz" integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== @@ -1191,14 +1206,6 @@ flatted@^3.1.0: resolved "https://registry.npmjs.org/flatted/-/flatted-3.2.2.tgz" integrity sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA== -fluent-ffmpeg@^2.1.2: - version "2.1.2" - resolved "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz" - integrity sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ= - dependencies: - async ">=0.2.9" - which "^1.1.1" - follow-redirects@^1.14.8, follow-redirects@^1.14.9: version "1.15.0" resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.0.tgz" @@ -1572,6 +1579,11 @@ is-unix@~2.0.1: resolved "https://registry.npmjs.org/is-unix/-/is-unix-2.0.1.tgz" integrity sha512-RyKp5JtlRnfOvnKtfBMPLw9ocqDe1NlPQ8Bt+geVzKGfMnLGj8z/Y2HOmk/aMf47P4EbrEt9dN6YGTT1fx4mZA== +is-url@^1.2.4: + version "1.2.4" + resolved "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz" + integrity sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww== + is-wsl@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz" @@ -1905,7 +1917,7 @@ node-cache@^5.1.2: dependencies: clone "2.x" -node-fetch@~2.6.5: +node-fetch@^2.6.1, node-fetch@~2.6.5: version "2.6.7" resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== @@ -2264,6 +2276,11 @@ redstar@0.0.2: dependencies: minimatch "~3.0.4" +regenerator-runtime@^0.13.7: + version "0.13.9" + resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz" + integrity sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA== + regexpp@^3.2.0: version "3.2.0" resolved "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz" @@ -2284,6 +2301,11 @@ resolve-from@^4.0.0: resolved "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz" integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== +resolve-url@^0.2.1: + version "0.2.1" + resolved "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz" + integrity sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg== + responselike@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz" @@ -2304,7 +2326,7 @@ reusify@^1.0.4: resolved "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz" integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== -rimraf@^3.0.0, rimraf@^3.0.2: +rimraf@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz" integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== @@ -2570,13 +2592,6 @@ timers-ext@^0.1.7: es5-ext "~0.10.46" next-tick "1" -tmp@^0.2.1: - version "0.2.1" - resolved "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz" - integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== - dependencies: - rimraf "^3.0.0" - to-regex-range@^5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz" @@ -2694,13 +2709,6 @@ whatwg-url@^5.0.0: tr46 "~0.0.3" webidl-conversions "^3.0.0" -which@^1.1.1: - version "1.3.1" - resolved "https://registry.npmjs.org/which/-/which-1.3.1.tgz" - integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== - dependencies: - isexe "^2.0.0" - which@^2.0.1: version "2.0.2" resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz"