feat: replace native ffmpeg with bundled wasm version (#305)

This commit is contained in:
Miraculous Owonubi 2022-09-04 06:12:27 +04:00 committed by GitHub
parent 19a38347f0
commit 890a1dd9a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 405 additions and 338 deletions

View File

@ -4,6 +4,9 @@
"es6": true,
"node": true
},
"globals": {
"globalThis": false
},
"parserOptions": {
"sourceType": "module",
"ecmaVersion": "latest",

View File

@ -5,7 +5,6 @@ on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
schedule:
- cron: '0 0 * * WED,SAT' # 00:00 on Wednesdays and Saturdays, weekly.
@ -120,24 +119,31 @@ jobs:
- name: Checkout Repository
uses: actions/checkout@v3
- name: Get Git SHAs
id: get-shas
if: github.event_name == 'pull_request'
run: |
BASE_SHA=$( echo ${{ github.event.pull_request.base.sha }} | head -c7 )
echo "::set-output name=base_sha::$BASE_SHA"
HEAD_SHA=$( echo ${{ github.event.pull_request.head.sha }} | head -c7 )
echo "::set-output name=head_sha::$HEAD_SHA"
- name: Extract metadata (tags, labels) for Docker
id: meta
id: docker-meta
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681
with:
images: freyrcli/freyrjs-git
tags: |
type=ref,event=pr
type=ref,event=branch
type=sha,format=short,prefix=
type=raw,value=${{ steps.get-shas.outputs.head_sha }}
- name: Extract Tags
id: meta2
id: get-docker-tag
if: github.event_name == 'pull_request'
run: |
PR_TAG=$( echo "${{ steps.meta.outputs.tags }}" | sed 's/freyrcli\/freyrjs-git://g' )
PR_TAG=$( echo "${{ steps.docker-meta.outputs.tags }}" | sed 's/freyrcli\/freyrjs-git://g' )
echo "::set-output name=tag::$PR_TAG"
SHA=$( echo ${{ github.event.pull_request.base.sha }} | head -c7 )
echo "::set-output name=sha::$SHA"
- name: Report Docker Image Build Status
uses: marocchino/sticky-pull-request-comment@39c5b5dc7717447d0cba270cd115037d32d28443
@ -155,10 +161,10 @@ jobs:
**A docker image for this PR is being built!**
```console
docker pull freyrcli/freyrjs-git:${{steps.meta2.outputs.tag}}
docker pull freyrcli/freyrjs-git:${{ steps.get-docker-tag.outputs.tag }}
```
| [**Base Branch (${{ github.event.pull_request.base.ref }})**][base-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{steps.meta2.outputs.sha}}?color=gray&label=%20&logo=docker)][base-url] |
| [**Base Branch (${{ github.event.pull_request.base.ref }})**][base-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{ steps.get-shas.outputs.base_sha }}?color=gray&label=%20&logo=docker)][base-url] |
| :-: | - |
---
@ -173,7 +179,7 @@ jobs:
</details>
</div>
[base-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{steps.meta2.outputs.sha}}
[base-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{ steps.get-shas.outputs.base_sha }}
- name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8
@ -193,8 +199,8 @@ jobs:
context: .
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
tags: ${{ steps.docker-meta.outputs.tags }}
labels: ${{ steps.docker-meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
@ -212,12 +218,12 @@ jobs:
**A docker image for this PR has been built!**
```console
docker pull freyrcli/freyrjs-git:${{steps.meta2.outputs.tag}}
docker pull freyrcli/freyrjs-git:${{ steps.get-docker-tag.outputs.tag }}
```
| [**Base Branch (${{ github.event.pull_request.base.ref }})**][base-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{steps.meta2.outputs.sha}}?color=gray&label=%20&logo=docker)][base-url] |
| [**Base Branch (${{ github.event.pull_request.base.ref }})**][base-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{ steps.get-shas.outputs.base_sha }}?color=gray&label=%20&logo=docker)][base-url] |
| :-: | - |
| [**This Patch**][pr-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{steps.meta2.outputs.tag}}?color=gray&label=%20&logo=docker)][pr-url] |
| [**This Patch**][pr-url] | [![](https://img.shields.io/docker/image-size/freyrcli/freyrjs-git/${{ steps.get-docker-tag.outputs.tag }}?color=gray&label=%20&logo=docker)][pr-url] |
[![][compare-img]][compare-url]
@ -233,10 +239,10 @@ jobs:
</details>
</div>
[base-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{steps.meta2.outputs.sha}}
[pr-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{steps.meta2.outputs.tag}}
[base-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{ steps.get-shas.outputs.base_sha }}
[pr-url]: https://hub.docker.com/r/freyrcli/freyrjs-git/tags?name=${{ steps.get-docker-tag.outputs.tag }}
[compare-img]: https://img.shields.io/badge/%20-compare-gray?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+CiAgIDxwYXRoIGZpbGw9IiNmZmZmZmYiIGQ9Ik0zLDFDMS44OSwxIDEsMS44OSAxLDNWMTRDMSwxNS4xMSAxLjg5LDE2IDMsMTZINVYxNEgzVjNIMTRWNUgxNlYzQzE2LDEuODkgMTUuMTEsMSAxNCwxSDNNOSw3QzcuODksNyA3LDcuODkgNyw5VjExSDlWOUgxMVY3SDlNMTMsN1Y5SDE0VjEwSDE2VjdIMTNNMTgsN1Y5SDIwVjIwSDlWMThIN1YyMEM3LDIxLjExIDcuODksMjIgOSwyMkgyMEMyMS4xMSwyMiAyMiwyMS4xMSAyMiwyMFY5QzIyLDcuODkgMjEuMTEsNyAyMCw3SDE4TTE0LDEyVjE0SDEyVjE2SDE0QzE1LjExLDE2IDE2LDE1LjExIDE2LDE0VjEySDE0TTcsMTNWMTZIMTBWMTRIOVYxM0g3WiIgLz4KPC9zdmc+
[compare-url]: https://portal.slim.dev/home/diff/dockerhub%3A%2F%2Fdockerhub.public%2Ffreyrcli%2Ffreyrjs-git%3A${{steps.meta2.outputs.tag}}#file-system
[compare-url]: https://portal.slim.dev/home/diff/dockerhub%3A%2F%2Fdockerhub.public%2Ffreyrcli%2Ffreyrjs-git%3A${{ steps.get-docker-tag.outputs.tag }}#file-system
linter:
runs-on: ubuntu-latest

View File

@ -21,7 +21,7 @@ RUN go install github.com/tj/node-prune@1159d4c \
FROM alpine:3.16.2 as base
# hadolint ignore=DL3018
RUN apk add --no-cache nodejs ffmpeg python3 \
RUN apk add --no-cache nodejs python3 \
&& find /usr/lib/python3* \
\( -type d -name __pycache__ -o -type f -name '*.whl' \) \
-exec rm -r {} \+

View File

@ -130,25 +130,6 @@ Here's a list of the metadata that freyr can extract from each streaming service
</details>
<details>
<summary>ffmpeg >= v0.9</summary>
<!-- textlint-disable -->
Download for your individual platforms here <https://ffmpeg.org/download.html>
<!-- textlint-enable -->
- Windows + macOS:
- Ensure to extract the `ffmpeg` binary from the compressed file, if it's in one.
- make sure it's available in your `PATH`
- otherwise, set `FFMPEG_PATH` to explicitly specify binary to use
- Linux: _(check individual package managers)_
- Debian: The `ppa:mc3man/trusty-media` PPA provides recent builds
- Arch Linux: `sudo pacman -S ffmpeg`
- Android (Termux): `apt install ffmpeg`
- Alpine Linux: `sudo apk add ffmpeg`
</details>
<details>
<summary>AtomicParsley >= (v0.9.6 | 20200701)</summary>
@ -315,7 +296,6 @@ Options:
--no-auth skip authentication procedure
--no-browser disable auto-launching of user browser
--no-net-check disable internet connection check
--ffmpeg <PATH> explicit path to the ffmpeg binary
--atomic-parsley <PATH> explicit path to the atomic-parsley binary
--no-stats don't show the stats on completion
--pulsate-bar show a pulsating bar
@ -325,7 +305,6 @@ Options:
Environment Variables:
SHOW_DEBUG_STACK show extended debug information
FFMPEG_PATH custom ffmpeg path, alternatively use `--ffmpeg`
ATOMIC_PARSLEY_PATH custom AtomicParsley path, alternatively use `--atomic-parsley`
Info:

382
cli.js
View File

@ -9,7 +9,6 @@ import {promises as fs, constants as fs_constants, createReadStream, createWrite
import Conf from 'conf';
import open from 'open';
import xget from 'libxget';
import ffmpeg from 'fluent-ffmpeg';
import merge2 from 'merge2';
import mkdirp from 'mkdirp';
import xbytes from 'xbytes';
@ -26,6 +25,7 @@ import {isBinaryFile} from 'isbinaryfile';
import {fileTypeFromFile} from 'file-type';
import {program as commander} from 'commander';
import {decode as entityDecode} from 'html-entities';
import {createFFmpeg, fetchFile} from '@ffmpeg/ffmpeg';
import _merge from 'lodash.merge';
import _mergeWith from 'lodash.mergewith';
@ -633,13 +633,6 @@ async function init(packageJson, queries, options) {
let atomicParsley;
try {
if (options.ffmpeg) {
if (!(await maybeStat(options.ffmpeg))) throw new Error(`\x1b[31mffmpeg\x1b[0m: Binary not found [${options.ffmpeg}]`);
if (!(await isBinaryFile(options.ffmpeg)))
stackLogger.warn('\x1b[33mffmpeg\x1b[0m: Detected non-binary file, trying anyways...');
ffmpeg.setFfmpegPath(options.ffmpeg);
}
if (options.atomicParsley) {
if (!(await maybeStat(options.atomicParsley)))
throw new Error(`\x1b[31mAtomicParsley\x1b[0m: Binary not found [${options.atomicParsley}]`);
@ -654,7 +647,7 @@ async function init(packageJson, queries, options) {
async function createPlaylist(header, stats, logger, filename, playlistTitle, shouldAppend) {
if (options.playlist !== false) {
const validStats = stats.filter(stat => (stat.code === 0 ? stat.complete : !stat.code));
const validStats = stats.filter(stat => (stat[symbols.errorCode] === 0 ? stat.complete : !stat[symbols.errorCode]));
if (validStats.length) {
logger.print('[\u2022] Creating playlist...');
const playlistFile = xpath.join(
@ -704,9 +697,8 @@ async function init(packageJson, queries, options) {
function downloadToStream({urlOrFragments, outputFile, logger, opts}) {
opts = {tag: '', successMessage: '', failureMessage: '', retryMessage: '', ...opts};
[opts.tag, opts.errorHandler, opts.retryMessage, opts.failureMessage, opts.successMessage, opts.altMessage] = [
[opts.tag, opts.retryMessage, opts.failureMessage, opts.successMessage, opts.altMessage] = [
opts.tag,
opts.errorHandler,
opts.retryMessage,
opts.failureMessage,
opts.successMessage,
@ -748,7 +740,6 @@ async function init(packageJson, queries, options) {
if (!options.bar) logger.write('\x1b[G\x1b[K');
logger.write(opts.failureMessage(err), '\n');
}
opts.errorHandler(err);
rej(err);
});
@ -822,7 +813,6 @@ async function init(packageJson, queries, options) {
logger.write('\x1b[G\x1b[K');
logger.write(opts.failureMessage(err), '\n');
}
opts.errorHandler(err);
rej(err);
});
return !options.bar ? feed : feed.pipe(barGen.next(frag.size));
@ -848,70 +838,88 @@ async function init(packageJson, queries, options) {
Config.concurrency.downloader,
async ({track, meta, feedMeta, trackLogger}) => {
const baseCacheDir = 'fr3yrcach3';
const imageFile = await fileMgr({
filename: `freyrcli-${meta.fingerprint}.x4i`,
tempdir: Config.dirs.cacheDir === '<tmp>' ? undefined : Config.dirs.cacheDir,
dirname: baseCacheDir,
keep: true,
});
const imageBytesWritten = await downloadToStream({
urlOrFragments: track.getImage(Config.image.width, Config.image.height),
outputFile: imageFile.path,
logger: trackLogger,
opts: {
tag: '[Retrieving album art]...',
errorHandler: () => imageFile.removeCallback(),
retryMessage: data => trackLogger.getText(`| ${getRetryMessage(data)}`),
resumeHandler: offset => trackLogger.log(cStringd(`| :{color(yellow)}{i}:{color:close(yellow)} Resuming at ${offset}`)),
failureMessage: err =>
trackLogger.getText(`| [\u2715] Failed to get album art${err ? ` [${err.code || err.message}]` : ''}`),
successMessage: trackLogger.getText(`| [\u2713] Got album art`),
altMessage: trackLogger.getText('| \u27a4 Downloading album art...'),
},
}).catch(err => Promise.reject({err, code: 3}));
let imageBytesWritten = 0;
try {
imageBytesWritten = await downloadToStream({
urlOrFragments: track.getImage(Config.image.width, Config.image.height),
outputFile: imageFile.path,
logger: trackLogger,
opts: {
tag: '[Retrieving album art]...',
retryMessage: data => trackLogger.getText(`| ${getRetryMessage(data)}`),
resumeHandler: offset =>
trackLogger.log(cStringd(`| :{color(yellow)}{i}:{color:close(yellow)} Resuming at ${offset}`)),
failureMessage: err =>
trackLogger.getText(`| [\u2715] Failed to get album art${err ? ` [${err.code || err.message}]` : ''}`),
successMessage: trackLogger.getText(`| [\u2713] Got album art`),
altMessage: trackLogger.getText('| \u27a4 Downloading album art...'),
},
});
} catch (err) {
await imageFile.removeCallback();
throw {err, [symbols.errorCode]: 3};
}
const rawAudio = await fileMgr({
filename: `freyrcli-${meta.fingerprint}.x4a`,
tempdir: Config.dirs.cacheDir === '<tmp>' ? undefined : Config.dirs.cacheDir,
dirname: baseCacheDir,
keep: true,
});
const audioBytesWritten = await downloadToStream(
_merge(
{
outputFile: rawAudio.path,
logger: trackLogger,
opts: {
tag: `[${meta.trackName}]`,
retryMessage: data => trackLogger.getText(`| ${getRetryMessage(data)}`),
resumeHandler: offset =>
trackLogger.log(cStringd(`| :{color(yellow)}{i}:{color:close(yellow)} Resuming at ${offset}`)),
successMessage: trackLogger.getText('| [\u2713] Got raw track file'),
altMessage: trackLogger.getText('| \u27a4 Downloading track...'),
},
},
feedMeta.protocol !== 'http_dash_segments'
? {
urlOrFragments: feedMeta.url,
opts: {
errorHandler: () => rawAudio.removeCallback(),
failureMessage: err =>
trackLogger.getText(`| [\u2715] Failed to get raw media stream${err ? ` [${err.code || err.message}]` : ''}`),
},
}
: {
urlOrFragments: feedMeta.fragments.map(({path}) => ({
url: `${feedMeta.fragment_base_url}${path}`,
...(([, min, max]) => ({min: +min, max: +max, size: +max - +min + 1}))(path.match(/range\/(\d+)-(\d+)$/)),
})),
opts: {
failureMessage: err =>
trackLogger.getText(
`| [\u2715] Segment error while getting raw media${err ? ` [${err.code || err.message}]` : ''}`,
),
},
let audioBytesWritten = 0;
try {
audioBytesWritten = await downloadToStream(
_merge(
{
outputFile: rawAudio.path,
logger: trackLogger,
opts: {
tag: `[${meta.trackName}]`,
retryMessage: data => trackLogger.getText(`| ${getRetryMessage(data)}`),
resumeHandler: offset =>
trackLogger.log(cStringd(`| :{color(yellow)}{i}:{color:close(yellow)} Resuming at ${offset}`)),
successMessage: trackLogger.getText('| [\u2713] Got raw track file'),
altMessage: trackLogger.getText('| \u27a4 Downloading track...'),
},
),
).catch(err => Promise.reject({err, code: 4}));
},
feedMeta.protocol !== 'http_dash_segments'
? {
urlOrFragments: feedMeta.url,
opts: {
failureMessage: err =>
trackLogger.getText(
`| [\u2715] Failed to get raw media stream${err ? ` [${err.code || err.message}]` : ''}`,
),
},
}
: {
urlOrFragments: feedMeta.fragments.map(({path}) => ({
url: `${feedMeta.fragment_base_url}${path}`,
...(([, min, max]) => ({min: +min, max: +max, size: +max - +min + 1}))(path.match(/range\/(\d+)-(\d+)$/)),
})),
opts: {
failureMessage: err =>
trackLogger.getText(
`| [\u2715] Segment error while getting raw media${err ? ` [${err.code || err.message}]` : ''}`,
),
},
},
),
);
} catch (err) {
await rawAudio.removeCallback();
throw {err, [symbols.errorCode]: 4};
}
return {
image: {file: imageFile, bytesWritten: imageBytesWritten},
audio: {file: rawAudio, bytesWritten: audioBytesWritten},
@ -923,94 +931,124 @@ async function init(packageJson, queries, options) {
'cli:postprocessor:embedQueue',
Config.concurrency.embedder,
async ({track, meta, files, audioSource}) => {
return Promise.promisify(atomicParsley)(meta.outFilePath, {
overWrite: '', // overwrite the file
try {
await Promise.promisify(atomicParsley)(meta.outFilePath, {
overWrite: '', // overwrite the file
title: track.name, // ©nam
artist: track.artists[0], // ©ART
composer: track.composers, // ©wrt
album: track.album, // ©alb
genre: (genre => (genre ? genre.concat(' ') : ''))((track.genres || [])[0]), // ©gen | gnre
tracknum: `${track.track_number}/${track.total_tracks}`, // trkn
disk: `${track.disc_number}/${track.disc_number}`, // disk
year: new Date(track.release_date).toISOString().split('T')[0], // ©day
compilation: track.compilation, // ©cpil
gapless: options.gapless, // pgap
rDNSatom: [
// ----
['Digital Media', 'name=MEDIA', 'domain=com.apple.iTunes'],
[track.isrc, 'name=ISRC', 'domain=com.apple.iTunes'],
[track.artists[0], 'name=ARTISTS', 'domain=com.apple.iTunes'],
[track.label, 'name=LABEL', 'domain=com.apple.iTunes'],
[`${meta.service[symbols.meta].DESC}: ${track.uri}`, 'name=SOURCE', 'domain=com.apple.iTunes'],
[
`${audioSource.service[symbols.meta].DESC}: ${audioSource.source.videoId}`,
'name=PROVIDER',
'domain=com.apple.iTunes',
title: track.name, // ©nam
artist: track.artists[0], // ©ART
composer: track.composers, // ©wrt
album: track.album, // ©alb
genre: (genre => (genre ? genre.concat(' ') : ''))((track.genres || [])[0]), // ©gen | gnre
tracknum: `${track.track_number}/${track.total_tracks}`, // trkn
disk: `${track.disc_number}/${track.disc_number}`, // disk
year: new Date(track.release_date).toISOString().split('T')[0], // ©day
compilation: track.compilation, // ©cpil
gapless: options.gapless, // pgap
rDNSatom: [
// ----
['Digital Media', 'name=MEDIA', 'domain=com.apple.iTunes'],
[track.isrc, 'name=ISRC', 'domain=com.apple.iTunes'],
[track.artists[0], 'name=ARTISTS', 'domain=com.apple.iTunes'],
[track.label, 'name=LABEL', 'domain=com.apple.iTunes'],
[`${meta.service[symbols.meta].DESC}: ${track.uri}`, 'name=SOURCE', 'domain=com.apple.iTunes'],
[
`${audioSource.service[symbols.meta].DESC}: ${audioSource.source.videoId}`,
'name=PROVIDER',
'domain=com.apple.iTunes',
],
],
],
advisory: ['explicit', 'clean'].includes(track.contentRating) // rtng
? track.contentRating
: track.contentRating === true
? 'explicit'
: 'Inoffensive',
stik: 'Normal', // stik
// geID: 0, // geID: genreID. See `AtomicParsley --genre-list`
// sfID: 0, // ~~~~: store front ID
// cnID: 0, // cnID: catalog ID
albumArtist: track.album_artist, // aART
// ownr? <owner>
purchaseDate: 'timestamp', // purd
apID: 'cli@freyr.git', // apID
copyright: track.copyrights.sort(({type}) => (type === 'P' ? -1 : 1))[0].text, // cprt
encodingTool: `freyr-js cli v${packageJson.version}`, // ©too
encodedBy: 'd3vc0dr', // ©enc
artwork: files.image.file.path, // covr
// sortOrder: [
// ['name', 'NAME'], // sonm
// ['album', 'NAME'], // soal
// ['artist', 'NAME'], // soar
// ['albumartist', 'NAME'], // soaa
// ],
})
.finally(() => files.image.file.removeCallback())
.catch(err => Promise.reject({err, code: 8}));
advisory: ['explicit', 'clean'].includes(track.contentRating) // rtng
? track.contentRating
: track.contentRating === true
? 'explicit'
: 'Inoffensive',
stik: 'Normal', // stik
// geID: 0, // geID: genreID. See `AtomicParsley --genre-list`
// sfID: 0, // ~~~~: store front ID
// cnID: 0, // cnID: catalog ID
albumArtist: track.album_artist, // aART
// ownr? <owner>
purchaseDate: 'timestamp', // purd
apID: 'cli@freyr.git', // apID
copyright: track.copyrights.sort(({type}) => (type === 'P' ? -1 : 1))[0].text, // cprt
encodingTool: `freyr-js cli v${packageJson.version}`, // ©too
encodedBy: 'd3vc0dr', // ©enc
artwork: files.image.file.path, // covr
// sortOrder: [
// ['name', 'NAME'], // sonm
// ['album', 'NAME'], // soal
// ['artist', 'NAME'], // soar
// ['albumartist', 'NAME'], // soaa
// ],
});
} catch (err) {
throw {err, [symbols.errorCode]: 8};
} finally {
await files.image.file.removeCallback();
}
},
);
delete globalThis.fetch;
const encodeQueue = new AsyncQueue(
'cli:postprocessor:encodeQueue',
Config.concurrency.encoder,
async ({track, meta, files}) => {
return new Promise((res, rej) =>
ffmpeg()
.addInput(files.audio.file.path)
.audioCodec('aac')
.audioBitrate(options.bitrate)
.audioFrequency(44100)
.noVideo()
.setDuration(TimeFormat.fromMs(track.duration, 'hh:mm:ss.sss'))
.toFormat('ipod')
.saveToFile(meta.outFilePath)
.on('error', err => rej({err, code: 7}))
.on('end', res),
).finally(() => files.audio.file.removeCallback());
},
AsyncQueue.provision(
async () => {
let ffmpeg = createFFmpeg({log: false});
await ffmpeg.load();
return ffmpeg;
},
async (ffmpeg, {track, meta, files}) => {
let infile = xpath.basename(files.audio.file.path);
let outfile = xpath.basename(files.audio.file.path.replace(/\.x4a$/, '.m4a'));
try {
ffmpeg.FS('writeFile', infile, await fetchFile(files.audio.file.path));
await ffmpeg.run(
'-i',
infile,
'-acodec',
'aac',
'-b:a',
options.bitrate,
'-ar',
'44100',
'-vn',
'-t',
TimeFormat.fromMs(track.duration, 'hh:mm:ss.sss'),
'-f',
'ipod',
outfile,
);
await fs.writeFile(meta.outFilePath, ffmpeg.FS('readFile', outfile));
} catch (err) {
throw {err, [symbols.errorCode]: 7};
} finally {
await files.audio.file.removeCallback();
}
},
),
);
const postProcessor = new AsyncQueue('cli:postProcessor', 4, async ({track, meta, files, audioSource}) => {
await mkdirp(meta.outFileDir).catch(err => Promise.reject({err, code: 6}));
const wroteImage =
!!options.cover &&
(await (async outArtPath =>
(await maybeStat(outArtPath).then(stat => stat && stat.isFile())) ||
(await fs.copyFile(files.image.file.path, outArtPath), true))(
xpath.join(meta.outFileDir, `${options.cover}.${(await fileTypeFromFile(files.image.file.path)).ext}`),
));
await encodeQueue.push({track, meta, files});
await embedQueue.push({track, meta, files, audioSource});
return {wroteImage, finalSize: (await fs.stat(meta.outFilePath)).size};
});
const postProcessor = new AsyncQueue(
'cli:postProcessor',
Math.max(Config.concurrency.encoder, Config.concurrency.embedder),
async ({track, meta, files, audioSource}) => {
await mkdirp(meta.outFileDir).catch(err => Promise.reject({err, [symbols.errorCode]: 6}));
const wroteImage =
!!options.cover &&
(await (async outArtPath =>
(await maybeStat(outArtPath).then(stat => stat && stat.isFile())) ||
(await fs.copyFile(files.image.file.path, outArtPath), true))(
xpath.join(meta.outFileDir, `${options.cover}.${(await fileTypeFromFile(files.image.file.path)).ext}`),
));
await encodeQueue.push({track, meta, files});
await embedQueue.push({track, meta, files, audioSource});
return {wroteImage, finalSize: (await fs.stat(meta.outFilePath)).size};
},
);
function buildSourceCollectorFor(track, selector) {
async function handleSource(iterator, lastErr) {
@ -1065,13 +1103,13 @@ async function init(packageJson, queries, options) {
const filterStat = options.filter(track, false);
if (!filterStat.status) {
trackLogger.log("| [\u2022] Didn't match filter. Skipping...");
return {meta, code: 0, skip_reason: `filtered out: ${filterStat.reason.message}`, complete: false};
return {meta, [symbols.errorCode]: 0, skip_reason: `filtered out: ${filterStat.reason.message}`, complete: false};
}
if (props.fileExists) {
if (!props.processTrack) {
trackLogger.log('| [\u00bb] Track exists. Skipping...');
return {meta, code: 0, skip_reason: 'exists', complete: true};
return {meta, [symbols.errorCode]: 0, skip_reason: 'exists', complete: true};
}
trackLogger.log('| [\u2022] Track exists. Overwriting...');
}
@ -1083,12 +1121,12 @@ async function init(packageJson, queries, options) {
onPass: ({sources}) => `[success, found ${sources.length} source${sources.length === 1 ? '' : 's'}]\n`,
}),
);
if ('err' in audioSource) return {meta, code: 1, err: audioSource.err}; // zero sources found
if ('err' in audioSource) return {meta, [symbols.errorCode]: 1, err: audioSource.err}; // zero sources found
const audioFeeds = await processPromise(audioSource.feeds, trackLogger, {
onInit: '| \u27a4 Awaiting audiofeeds...',
noVal: () => '[Unable to collect source feeds]\n',
});
if (!audioFeeds || audioFeeds.err) return {meta, err: (audioFeeds || {}).err, code: 2};
if (!audioFeeds || audioFeeds.err) return {meta, err: (audioFeeds || {}).err, [symbols.errorCode]: 2};
const [feedMeta] = audioFeeds.formats
.filter(meta => 'abr' in meta && !('vbr' in meta))
@ -1097,11 +1135,15 @@ async function init(packageJson, queries, options) {
meta.fingerprint = crypto.createHash('md5').update(`${audioSource.source.videoId} ${feedMeta.format_id}`).digest('hex');
const files = await downloadQueue
.push({track, meta, feedMeta, trackLogger})
.catch(errObject => Promise.reject({meta, code: 5, ...(errObject.code ? errObject : {err: errObject})}));
.catch(errObject =>
Promise.reject({meta, [symbols.errorCode]: 5, ...(symbols.errorCode in errObject ? errObject : {err: errObject})}),
);
trackLogger.log(`| [\u2022] Post Processing...`);
return {
files,
postprocess: postProcessor.push({track, meta, files, audioSource}).catch(errObject => ({code: 9, ...errObject})),
postprocess: postProcessor
.push({track, meta, files, audioSource})
.catch(errObject => ({[symbols.errorCode]: 9, ...(symbols.errorCode in errObject ? errObject : {err: errObject})})),
};
});
@ -1112,11 +1154,11 @@ async function init(packageJson, queries, options) {
try {
if (!(track = await track)) throw new Error('no data recieved from track');
} catch (err) {
return {code: -1, err};
return {[symbols.errorCode]: -1, err};
}
if ((track[symbols.errorStack] || {}).code === 1)
return {
code: -1,
[symbols.errorCode]: -1,
err: new Error("local-typed tracks aren't supported"),
meta: {track: {uri: track[symbols.errorStack].uri}},
};
@ -1140,7 +1182,7 @@ async function init(packageJson, queries, options) {
return trackQueue
.push({track, meta, props: {collectSources, fileExists, processTrack, logger}})
.then(trackObject => ({...trackObject, meta}))
.catch(errObject => ({meta, code: 10, ...errObject}));
.catch(errObject => ({meta, [symbols.errorCode]: 10, ...errObject}));
},
);
@ -1326,34 +1368,34 @@ async function init(packageJson, queries, options) {
await Promise.mapSeries(trackStats, async trackStat => {
if (trackStat.postprocess) {
trackStat.postprocess = await trackStat.postprocess;
if ('code' in trackStat.postprocess) {
trackStat.code = trackStat.postprocess.code;
if (symbols.errorCode in trackStat.postprocess) {
trackStat[symbols.errorCode] = trackStat.postprocess[symbols.errorCode];
trackStat.err = trackStat.postprocess.err;
}
}
if (trackStat.code) {
if (trackStat[symbols.errorCode]) {
const reason =
trackStat.code === -1
trackStat[symbols.errorCode] === -1
? 'Failed getting track data'
: trackStat.code === 1
: trackStat[symbols.errorCode] === 1
? 'Failed collecting sources'
: trackStat.code === 2
: trackStat[symbols.errorCode] === 2
? 'Error while collecting sources feeds'
: trackStat.code === 3
: trackStat[symbols.errorCode] === 3
? 'Error downloading album art'
: trackStat.code === 4
: trackStat[symbols.errorCode] === 4
? 'Error downloading raw audio'
: trackStat.code === 5
: trackStat[symbols.errorCode] === 5
? 'Unknown Download Error'
: trackStat.code === 6
: trackStat[symbols.errorCode] === 6
? 'Error ensuring directory integrity'
: trackStat.code === 7
: trackStat[symbols.errorCode] === 7
? 'Error while encoding audio'
: trackStat.code === 8
: trackStat[symbols.errorCode] === 8
? 'Failed while embedding metadata'
: trackStat.code === 9
? 'Unknown postprocessing error'
: 'Unknown track processing error';
: trackStat[symbols.errorCode] === 9
? 'Unexpected postprocessing error'
: 'Unexpected track processing error';
embedLogger.error(
`\u2022 [\u2715] ${trackStat.meta && trackStat.meta.trackName ? `${trackStat.meta.trackName}` : '<unknown track>'}${
trackStat.meta && trackStat.meta.track.uri ? ` [${trackStat.meta.track.uri}]` : ''
@ -1361,7 +1403,7 @@ async function init(packageJson, queries, options) {
trackStat.err ? ` [${trackStat.err['SHOW_DEBUG_STACK' in process.env ? 'stack' : 'message'] || trackStat.err}]` : ''
})`,
);
} else if (trackStat.code === 0)
} else if (trackStat[symbols.errorCode] === 0)
embedLogger.log(`\u2022 [\u00bb] ${trackStat.meta.trackName} (skipped: ${trackStat.skip_reason})`);
else
embedLogger.log(
@ -1408,10 +1450,10 @@ async function init(packageJson, queries, options) {
total.mediaSize += audio;
total.imageSize += image;
}
if (current.code === 0)
if (current[symbols.errorCode] === 0)
if (current.complete) total.passed += 1;
else total.skipped += 1;
else if (!('code' in current)) (total.new += 1), (total.passed += 1);
else if (!(symbols.errorCode in current)) (total.new += 1), (total.passed += 1);
else total.failed += 1;
return total;
},
@ -1554,7 +1596,6 @@ function prepCli(packageJson) {
.option('--no-browser', 'disable auto-launching of user browser')
.option('--no-net-check', 'disable internet connection check')
.option('--no-bar', 'disable the progress bar')
.option('--ffmpeg <PATH>', 'explicit path to the ffmpeg binary')
.option('--atomic-parsley <PATH>', 'explicit path to the atomic-parsley binary')
.option('--no-stats', "don't show the stats on completion")
.option('--pulsate-bar', 'show a pulsating bar')
@ -1570,7 +1611,6 @@ function prepCli(packageJson) {
console.log('');
console.log('Environment Variables:');
console.log(' SHOW_DEBUG_STACK show extended debug information');
console.log(' FFMPEG_PATH custom ffmpeg path, alternatively use `--ffmpeg`');
console.log(' ATOMIC_PARSLEY_PATH custom AtomicParsley path, alternatively use `--atomic-parsley`');
console.log('');
console.log('Info:');

133
package-lock.json generated
View File

@ -9,6 +9,8 @@
"version": "0.8.1",
"license": "Apache-2.0",
"dependencies": {
"@ffmpeg/core": "^0.10.0",
"@ffmpeg/ffmpeg": "^0.10.1",
"@yujinakayama/apple-music": "^0.4.1",
"async": "^3.2.4",
"bluebird": "^3.7.2",
@ -21,7 +23,6 @@
"express": "^4.18.1",
"file-type": "^18.0.0",
"filenamify": "^5.1.1",
"fluent-ffmpeg": "^2.1.2",
"got": "^12.1.0",
"hh-mm-ss": "^1.2.0",
"html-entities": "^2.3.3",
@ -42,7 +43,6 @@
"stringd": "^2.2.0",
"stringd-colors": "^1.10.0",
"stripchar": "^1.2.1",
"tmp": "^0.2.1",
"xbytes": "^1.8.0",
"xprogress": "^0.19.1",
"youtube-dl-exec": "^2.1.4",
@ -118,6 +118,25 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
"dev": true
},
"node_modules/@ffmpeg/core": {
"version": "0.10.0",
"resolved": "https://registry.npmjs.org/@ffmpeg/core/-/core-0.10.0.tgz",
"integrity": "sha512-qunWJl5PezpXEm31tb8Qu5z37B5KVA1VYZCpXchMhuAb3X9T7PuE3SlhOwphEoRhzaOa3lpofDfzihAUMFaVPQ=="
},
"node_modules/@ffmpeg/ffmpeg": {
"version": "0.10.1",
"resolved": "https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.10.1.tgz",
"integrity": "sha512-ChQkH7Rh57hmVo1LhfQFibWX/xqneolJKSwItwZdKPcLZuKigtYAYDIvB55pDfP17VtR1R77SxgkB2/UApB+Og==",
"dependencies": {
"is-url": "^1.2.4",
"node-fetch": "^2.6.1",
"regenerator-runtime": "^0.13.7",
"resolve-url": "^0.2.1"
},
"engines": {
"node": ">=12.16.1"
}
},
"node_modules/@humanwhocodes/config-array": {
"version": "0.10.4",
"resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.4.tgz",
@ -2010,18 +2029,6 @@
"integrity": "sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==",
"dev": true
},
"node_modules/fluent-ffmpeg": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz",
"integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=",
"dependencies": {
"async": ">=0.2.9",
"which": "^1.1.1"
},
"engines": {
"node": ">=0.8.0"
}
},
"node_modules/follow-redirects": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.0.tgz",
@ -2086,7 +2093,8 @@
"node_modules/fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
"dev": true
},
"node_modules/function-bind": {
"version": "1.1.1",
@ -2130,6 +2138,7 @@
"version": "7.1.7",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz",
"integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==",
"dev": true,
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
@ -2628,6 +2637,11 @@
"node": ">= 12"
}
},
"node_modules/is-url": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz",
"integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww=="
},
"node_modules/is-wsl": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
@ -3612,6 +3626,11 @@
"node": "*"
}
},
"node_modules/regenerator-runtime": {
"version": "0.13.9",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz",
"integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA=="
},
"node_modules/regexpp": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz",
@ -3646,6 +3665,12 @@
"node": ">=4"
}
},
"node_modules/resolve-url": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz",
"integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==",
"deprecated": "https://github.com/lydell/resolve-url#deprecated"
},
"node_modules/responselike": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
@ -4081,17 +4106,6 @@
"next-tick": "1"
}
},
"node_modules/tmp": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
"integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
"dependencies": {
"rimraf": "^3.0.0"
},
"engines": {
"node": ">=8.17.0"
}
},
"node_modules/to-regex-range": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
@ -4274,17 +4288,6 @@
"webidl-conversions": "^3.0.0"
}
},
"node_modules/which": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
"integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
"dependencies": {
"isexe": "^2.0.0"
},
"bin": {
"which": "bin/which"
}
},
"node_modules/window-size": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/window-size/-/window-size-1.1.1.tgz",
@ -4487,6 +4490,22 @@
}
}
},
"@ffmpeg/core": {
"version": "0.10.0",
"resolved": "https://registry.npmjs.org/@ffmpeg/core/-/core-0.10.0.tgz",
"integrity": "sha512-qunWJl5PezpXEm31tb8Qu5z37B5KVA1VYZCpXchMhuAb3X9T7PuE3SlhOwphEoRhzaOa3lpofDfzihAUMFaVPQ=="
},
"@ffmpeg/ffmpeg": {
"version": "0.10.1",
"resolved": "https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.10.1.tgz",
"integrity": "sha512-ChQkH7Rh57hmVo1LhfQFibWX/xqneolJKSwItwZdKPcLZuKigtYAYDIvB55pDfP17VtR1R77SxgkB2/UApB+Og==",
"requires": {
"is-url": "^1.2.4",
"node-fetch": "^2.6.1",
"regenerator-runtime": "^0.13.7",
"resolve-url": "^0.2.1"
}
},
"@humanwhocodes/config-array": {
"version": "0.10.4",
"resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.4.tgz",
@ -5924,15 +5943,6 @@
"integrity": "sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==",
"dev": true
},
"fluent-ffmpeg": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz",
"integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=",
"requires": {
"async": ">=0.2.9",
"which": "^1.1.1"
}
},
"follow-redirects": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.0.tgz",
@ -6345,6 +6355,11 @@
"resolved": "https://registry.npmjs.org/is-unix/-/is-unix-2.0.1.tgz",
"integrity": "sha512-RyKp5JtlRnfOvnKtfBMPLw9ocqDe1NlPQ8Bt+geVzKGfMnLGj8z/Y2HOmk/aMf47P4EbrEt9dN6YGTT1fx4mZA=="
},
"is-url": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz",
"integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww=="
},
"is-wsl": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
@ -7055,6 +7070,11 @@
}
}
},
"regenerator-runtime": {
"version": "0.13.9",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz",
"integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA=="
},
"regexpp": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz",
@ -7077,6 +7097,11 @@
"integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
"dev": true
},
"resolve-url": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz",
"integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg=="
},
"responselike": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
@ -7398,14 +7423,6 @@
"next-tick": "1"
}
},
"tmp": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz",
"integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==",
"requires": {
"rimraf": "^3.0.0"
}
},
"to-regex-range": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
@ -7543,14 +7560,6 @@
"webidl-conversions": "^3.0.0"
}
},
"which": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
"integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
"requires": {
"isexe": "^2.0.0"
}
},
"window-size": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/window-size/-/window-size-1.1.1.tgz",

View File

@ -51,6 +51,8 @@
},
"license": "Apache-2.0",
"dependencies": {
"@ffmpeg/core": "^0.10.0",
"@ffmpeg/ffmpeg": "^0.10.1",
"@yujinakayama/apple-music": "^0.4.1",
"async": "^3.2.4",
"bluebird": "^3.7.2",
@ -63,7 +65,6 @@
"express": "^4.18.1",
"file-type": "^18.0.0",
"filenamify": "^5.1.1",
"fluent-ffmpeg": "^2.1.2",
"got": "^12.1.0",
"hh-mm-ss": "^1.2.0",
"html-entities": "^2.3.3",
@ -84,7 +85,6 @@
"stringd": "^2.2.0",
"stringd-colors": "^1.10.0",
"stripchar": "^1.2.1",
"tmp": "^0.2.1",
"xbytes": "^1.8.0",
"xprogress": "^0.19.1",
"youtube-dl-exec": "^2.1.4",

View File

@ -65,6 +65,18 @@ export default class AsyncQueue {
}, concurrency || 1);
}
static provision(genFn, worker) {
let resources = [];
return async (...args) => {
let resource = resources.shift() || (await genFn());
try {
return await worker(resource, ...args);
} finally {
resources.push(resource);
}
};
}
#_register = (objects, meta, handler) => {
const promises = (Array.isArray(objects) ? objects : [[objects, meta]]).map(objectBlocks => {
const [data, args] = Array.isArray(objectBlocks) ? objectBlocks : [objectBlocks, meta];

View File

@ -1,11 +1,12 @@
import {join} from 'path';
import {join, resolve, dirname} from 'path';
import {tmpdir} from 'os';
import {createHash} from 'crypto';
import {promises as fs, constants as fs_constants} from 'fs';
import tmp from 'tmp';
import mkdirp from 'mkdirp';
import esMain from 'es-main';
const openfiles = {};
const removeCallbacks = [];
function garbageCollector() {
@ -23,49 +24,56 @@ function hookupListeners() {
export default async function genFile(opts) {
opts = opts || {};
if (opts.filename) {
opts.tmpdir = opts.tmpdir || tmpdir();
const dir = join(opts.tmpdir, opts.dirname || '.');
await mkdirp(dir);
const path = join(dir, opts.filename);
const fd = await fs.open(path, fs_constants.O_CREAT | opts.mode);
hookupListeners();
let closed = false;
const garbageHandler = async keep => {
if (closed) return;
await fd.close();
closed = true;
if (!keep) await fs.unlink(path);
};
removeCallbacks.push(garbageHandler.bind(opts.keep));
return {
fd,
let mode = fs_constants.O_CREAT | opts.mode;
const path = resolve(join('tmpdir' in opts ? opts['tmpdir'] : tmpdir(), opts.dirname || '.', opts.filename));
let id = createHash('md5').update(`Ξ${mode}${path}`).digest('hex');
let file = openfiles[id];
if (!file) {
await mkdirp(dirname(path));
file = openfiles[id] = {
path,
removeCallback: async () => {
await garbageHandler(false);
removeCallbacks.splice(removeCallbacks.indexOf(garbageHandler), 1);
},
handle: await fs.open(path, mode),
refs: 1,
closed: false,
keep: false,
};
}
return new Promise((res, rej) =>
tmp.file(opts, (err, name, fd, removeCallback) => (err ? rej(err) : res({fd, name, removeCallback}))),
);
} else file.refs += 1;
hookupListeners();
const garbageHandler = async keep => {
file.keep ||= keep !== undefined ? keep : opts.keep;
if ((file.refs = Math.max(0, file.refs - 1))) return;
if (file.closed) return;
file.closed = true;
delete openfiles[id];
await file.handle.close();
if (!file.keep) await fs.unlink(path);
};
removeCallbacks.push(garbageHandler);
return {
path,
handle: file.handle,
removeCallback: async () => {
await garbageHandler(false);
removeCallbacks.splice(removeCallbacks.indexOf(garbageHandler), 1);
},
};
}
async function test() {
const filename = 'freyr_mgr_temp_file';
async function testMgr() {
const file = await genFile({filename});
async function testMgr(args) {
const file = await genFile({filename, ...args});
console.log('mgr>', file);
file.removeCallback();
return file;
}
async function testTmp() {
const file = await genFile({name: filename});
console.log('tmp>', file);
file.removeCallback();
}
await testMgr();
await testTmp();
let a = await testMgr();
let b = await testMgr();
await testMgr({keep: true});
let d = await testMgr();
a.removeCallback();
b.removeCallback();
// c.removeCallback(); // calling this would negate the keep directive
d.removeCallback();
}
if (esMain(import.meta)) test().catch(err => console.error('cli>', err));

View File

@ -1,7 +1,9 @@
const meta = Symbol('FreyrServiceMeta');
const errorCode = Symbol('FreyrErrorCode');
const errorStack = Symbol('FreyrErrorStack');
export default {
meta,
errorCode,
errorStack,
};

View File

@ -117,7 +117,7 @@ async function run_tests(suite, args, i) {
mode: fs_constants.W_OK,
});
logFile.stream = createWriteStream(null, {fd: logFile.fd});
logFile.stream = createWriteStream(null, {fd: logFile.handle});
let logline = line => `${line}`;

View File

@ -17,6 +17,21 @@
minimatch "^3.1.2"
strip-json-comments "^3.1.1"
"@ffmpeg/core@^0.10.0":
version "0.10.0"
resolved "https://registry.npmjs.org/@ffmpeg/core/-/core-0.10.0.tgz"
integrity sha512-qunWJl5PezpXEm31tb8Qu5z37B5KVA1VYZCpXchMhuAb3X9T7PuE3SlhOwphEoRhzaOa3lpofDfzihAUMFaVPQ==
"@ffmpeg/ffmpeg@^0.10.1":
version "0.10.1"
resolved "https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.10.1.tgz"
integrity sha512-ChQkH7Rh57hmVo1LhfQFibWX/xqneolJKSwItwZdKPcLZuKigtYAYDIvB55pDfP17VtR1R77SxgkB2/UApB+Og==
dependencies:
is-url "^1.2.4"
node-fetch "^2.6.1"
regenerator-runtime "^0.13.7"
resolve-url "^0.2.1"
"@humanwhocodes/config-array@^0.10.4":
version "0.10.4"
resolved "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.4.tgz"
@ -294,7 +309,7 @@ async.util.restparam@0.5.2:
resolved "https://registry.npmjs.org/async.util.restparam/-/async.util.restparam-0.5.2.tgz"
integrity sha1-A+/r88Ane5ciDlJaunUPXgT8gM0=
async@>=0.2.9, async@^3.2.4:
async@^3.2.4:
version "3.2.4"
resolved "https://registry.npmjs.org/async/-/async-3.2.4.tgz"
integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==
@ -1191,14 +1206,6 @@ flatted@^3.1.0:
resolved "https://registry.npmjs.org/flatted/-/flatted-3.2.2.tgz"
integrity sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==
fluent-ffmpeg@^2.1.2:
version "2.1.2"
resolved "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz"
integrity sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=
dependencies:
async ">=0.2.9"
which "^1.1.1"
follow-redirects@^1.14.8, follow-redirects@^1.14.9:
version "1.15.0"
resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.0.tgz"
@ -1572,6 +1579,11 @@ is-unix@~2.0.1:
resolved "https://registry.npmjs.org/is-unix/-/is-unix-2.0.1.tgz"
integrity sha512-RyKp5JtlRnfOvnKtfBMPLw9ocqDe1NlPQ8Bt+geVzKGfMnLGj8z/Y2HOmk/aMf47P4EbrEt9dN6YGTT1fx4mZA==
is-url@^1.2.4:
version "1.2.4"
resolved "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz"
integrity sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==
is-wsl@^2.2.0:
version "2.2.0"
resolved "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz"
@ -1905,7 +1917,7 @@ node-cache@^5.1.2:
dependencies:
clone "2.x"
node-fetch@~2.6.5:
node-fetch@^2.6.1, node-fetch@~2.6.5:
version "2.6.7"
resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz"
integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
@ -2264,6 +2276,11 @@ redstar@0.0.2:
dependencies:
minimatch "~3.0.4"
regenerator-runtime@^0.13.7:
version "0.13.9"
resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz"
integrity sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==
regexpp@^3.2.0:
version "3.2.0"
resolved "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz"
@ -2284,6 +2301,11 @@ resolve-from@^4.0.0:
resolved "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz"
integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==
resolve-url@^0.2.1:
version "0.2.1"
resolved "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz"
integrity sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==
responselike@^2.0.0:
version "2.0.0"
resolved "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz"
@ -2304,7 +2326,7 @@ reusify@^1.0.4:
resolved "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz"
integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==
rimraf@^3.0.0, rimraf@^3.0.2:
rimraf@^3.0.2:
version "3.0.2"
resolved "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz"
integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==
@ -2570,13 +2592,6 @@ timers-ext@^0.1.7:
es5-ext "~0.10.46"
next-tick "1"
tmp@^0.2.1:
version "0.2.1"
resolved "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz"
integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==
dependencies:
rimraf "^3.0.0"
to-regex-range@^5.0.1:
version "5.0.1"
resolved "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz"
@ -2694,13 +2709,6 @@ whatwg-url@^5.0.0:
tr46 "~0.0.3"
webidl-conversions "^3.0.0"
which@^1.1.1:
version "1.3.1"
resolved "https://registry.npmjs.org/which/-/which-1.3.1.tgz"
integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==
dependencies:
isexe "^2.0.0"
which@^2.0.1:
version "2.0.2"
resolved "https://registry.npmjs.org/which/-/which-2.0.2.tgz"