mirror
/
freyr-js
mirror of https://github.com/miraclx/freyr-js
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

revamp test infra (#303)

This commit is contained in:
Miraculous Owonubi 2022-08-07 10:44:54 +01:00 committed by GitHub
parent cabf965a39
commit 5f939e2b85
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 146 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
.github/workflows/tests.yml vendored
View File

@ -9,6 +9,9 @@ on:
schedule: schedule:
- cron: '0 0 * * WED,SAT' # 00:00 on Wednesdays and Saturdays, weekly. - cron: '0 0 * * WED,SAT' # 00:00 on Wednesdays and Saturdays, weekly.
env:
DOCKER_ARGS: "--user root"
jobs: jobs:
up-to-date: up-to-date:
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -101,9 +104,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Spotify - Download Track - name: Spotify - Download Track
run: | run: npm test -- --docker freyr-dev spotify.track
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev spotify.track
spotify-album: spotify-album:
if: ${{ always() }} if: ${{ always() }}
@ -132,9 +133,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Spotify - Download Album - name: Spotify - Download Album
run: | run: npm test -- --docker freyr-dev spotify.album
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev spotify.album
spotify-artist: spotify-artist:
if: ${{ always() }} if: ${{ always() }}
@ -163,9 +162,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Spotify - Download Album - name: Spotify - Download Album
run: | run: npm test -- --docker freyr-dev spotify.artist
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev spotify.artist
spotify-playlist: spotify-playlist:
if: ${{ always() }} if: ${{ always() }}
@ -194,9 +191,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Spotify - Download Playlist - name: Spotify - Download Playlist
run: | run: npm test -- --docker freyr-dev spotify.playlist
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev spotify.playlist
apple-music: apple-music:
if: false if: false
@ -231,9 +226,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Apple Music - Download Track - name: Apple Music - Download Track
run: | run: npm test -- --docker freyr-dev apple_music.track
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev apple_music.track
apple-music-album: apple-music-album:
if: ${{ always() }} if: ${{ always() }}
@ -262,9 +255,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Apple Music - Download Album - name: Apple Music - Download Album
run: | run: npm test -- --docker freyr-dev apple_music.album
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev apple_music.album
apple-music-artist: apple-music-artist:
if: ${{ always() }} if: ${{ always() }}
@ -293,9 +284,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Apple Music - Download Artist - name: Apple Music - Download Artist
run: | run: npm test -- --docker freyr-dev apple_music.artist
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev apple_music.artist
apple-music-playlist: apple-music-playlist:
if: ${{ always() }} if: ${{ always() }}
@ -324,9 +313,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Apple Music - Download Playlist - name: Apple Music - Download Playlist
run: | run: npm test -- --docker freyr-dev apple_music.playlist
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev apple_music.playlist
deezer: deezer:
if: false if: false
@ -361,9 +348,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Deezer - Download Track - name: Deezer - Download Track
run: | run: npm test -- --docker freyr-dev deezer.track
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev deezer.track
deezer-album: deezer-album:
if: ${{ always() }} if: ${{ always() }}
@ -392,9 +377,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Deezer - Download Album - name: Deezer - Download Album
run: | run: npm test -- --docker freyr-dev deezer.album
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev deezer.album
deezer-artist: deezer-artist:
if: ${{ always() }} if: ${{ always() }}
@ -423,9 +406,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Deezer - Download Artist - name: Deezer - Download Artist
run: | run: npm test -- --docker freyr-dev deezer.artist
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev deezer.artist
deezer-playlist: deezer-playlist:
if: ${{ always() }} if: ${{ always() }}
@ -454,9 +435,7 @@ jobs:
docker image ls -a docker image ls -a
- name: Deezer - Download Playlist - name: Deezer - Download Playlist
run: | run: npm test -- --docker freyr-dev deezer.playlist
DOCKER_ARGS="-v $(pwd)/stage:/data --user root" \
npm run test -- --docker freyr-dev deezer.playlist
docker-publish: docker-publish:

18
TEST.md
View File

@ -6,34 +6,40 @@ freyr is bundled with its own flexibly customizable test runner.
- To run all tests - To run all tests
```console ```console
npm run test -- --all npm test -- --all
``` ```
- To run just Spotify tests - To run just Spotify tests
```console ```console
npm run test -- spotify npm test -- spotify
``` ```
- To run just Apple Music artist tests - To run just Apple Music artist tests
```console ```console
npm run test -- apple_music.artist npm test -- apple_music.artist
``` ```
- You can use a custom test suite (see the [default suite](https://github.com/miraclx/freyr-js/blob/master/test/default.json) for an example) - You can use a custom test suite (see the [default suite](https://github.com/miraclx/freyr-js/blob/master/test/default.json) for an example)
```console ```console
npm run test -- --all --suite ./special_cases.json npm test -- --all --suite ./special_cases.json
``` ```
- And optionally, you can run the tests inside a freyr docker container - And optionally, you can run the tests inside a freyr docker container
```console ```console
npm run test -- deezer --docker freyr-dev:latest npm test -- deezer --docker freyr-dev:latest
``` ```
## `npm run test -- --help` - You can customize the working directory for storing the tracks and logs
```console
npm test -- spotify.track --name run-1 --stage ./test-runs
```
## `npm test -- --help`
```console ```console
freyr-test freyr-test

4
cli.js
View File

@ -736,7 +736,7 @@ async function init(packageJson, queries, options) {
data.store.get('progressBar').print(opts.retryMessage({ref: data.index + 1, ...data})); data.store.get('progressBar').print(opts.retryMessage({ref: data.index + 1, ...data}));
else { else {
if (!options.bar) logger.write('\x1b[G\x1b[K'); if (!options.bar) logger.write('\x1b[G\x1b[K');
logger.write(opts.retryMessage(data), '\n'); logger.write(opts.retryMessage({ref: data.index + 1, ...data}), '\n');
} }
} }
}) })
@ -807,7 +807,7 @@ async function init(packageJson, queries, options) {
if (options.bar) barGen.print(data); if (options.bar) barGen.print(data);
else { else {
logger.write('\x1b[G\x1b[K'); logger.write('\x1b[G\x1b[K');
logger.write(data, '\n'); logger.write({ref: `${i}[${data.index + 1}]`, ...data}, '\n');
} }
} }
}) })

2
src/file_mgr.js
View File

@ -11,7 +11,6 @@ const removeCallbacks = [];
const open = promisify(fs.open); const open = promisify(fs.open);
const close = promisify(fs.close); const close = promisify(fs.close);
const exists = promisify(fs.exists);
const unlink = promisify(fs.unlink); const unlink = promisify(fs.unlink);
function garbageCollector() { function garbageCollector() {
@ -31,7 +30,6 @@ export default async function genFile(opts) {
opts = opts || {}; opts = opts || {};
if (opts.filename) { if (opts.filename) {
opts.tmpdir = opts.tmpdir || tmpdir(); opts.tmpdir = opts.tmpdir || tmpdir();
if (!(await exists(opts.tmpdir))) throw new Error('tmpdir does not exist');
const dir = join(opts.tmpdir, opts.dirname || '.'); const dir = join(opts.tmpdir, opts.dirname || '.');
await mkdirp(dir); await mkdirp(dir);
const name = join(dir, opts.filename); const name = join(dir, opts.filename);

191
test/index.js
View File

@ -1,13 +1,16 @@
import fs from 'fs'; import fs from 'fs';
import url from 'url'; import url from 'url';
import path from 'path';
import util from 'util'; import util from 'util';
import {randomUUID} from 'crypto'; import {tmpdir} from 'os';
import {randomBytes} from 'crypto';
import {PassThrough} from 'stream'; import {PassThrough} from 'stream';
import {spawn} from 'child_process'; import {spawn} from 'child_process';
import {relative, join, resolve} from 'path';
import fileMgr from '../src/file_mgr.js'; import fileMgr from '../src/file_mgr.js';
const exists = util.promisify(fs.exists);
const __dirname = url.fileURLToPath(new URL('.', import.meta.url)); const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
function sed(fn) { function sed(fn) {
@ -52,40 +55,65 @@ async function pRetry(tries, fn) {
return result; return result;
} }
async function run_tests(stage, args) { function short_path(path) {
let docker_image, i; let a = resolve(path);
if ((docker_image = !!~(i = args.indexOf('--docker'))) && !(docker_image = args.splice(i, 2)[1])) let b = relative(process.cwd(), path);
if (!['..', '/'].some(c => b.startsWith(c))) b = `./${b}`;
return a.length < b.length ? a : b;
}
const default_stage = join(tmpdir(), 'freyr-test');
async function run_tests(suite, args, i) {
let docker_image;
if (~(i = args.indexOf('--docker')) && !(docker_image = args.splice(i, 2)[1]))
throw new Error('`--docker` requires an image name'); throw new Error('`--docker` requires an image name');
let stage_name = randomBytes(6).toString('hex');
if (~(i = args.indexOf('--name')) && !(stage_name = args.splice(i, 2)[1])) throw new Error('`--name` requires a stage name');
let stage_path = default_stage;
if (~(i = args.indexOf('--stage')) && !(stage_path = args.splice(i, 2)[1])) throw new Error('`--stage` requires a path');
stage_path = resolve(join(stage_path, stage_name));
let force, clean;
if ((force = !!~(i = args.indexOf('--force')))) args.splice(i, 1);
if ((clean = !!~(i = args.indexOf('--clean')))) args.splice(i, 1);
if (await exists(stage_path))
if (!force) throw new Error(`stage path [${stage_path}] already exists`);
else if (clean) fs.rmdirSync(stage_path);
let is_gha = 'GITHUB_ACTIONS' in process.env && process.env['GITHUB_ACTIONS'] === 'true'; let is_gha = 'GITHUB_ACTIONS' in process.env && process.env['GITHUB_ACTIONS'] === 'true';
if (~(i = args.indexOf('--all'))) args = Object.keys(stage);
let tests = args;
if (~(i = args.indexOf('--all'))) args.splice(i, 1), (tests = Object.keys(suite));
let invalidArg; let invalidArg;
if ((invalidArg = args.find(arg => arg.startsWith('-')))) throw new Error(`Invalid argument: ${invalidArg}`); if ((invalidArg = args.find(arg => arg.startsWith('-')))) throw new Error(`Invalid argument: ${invalidArg}`);
for (let [i, arg] of args.entries()) { if (!tests.length) return noService;
let [service, type] = arg.split('.');
if (!(service in stage)) throw new Error(`Invalid service: ${service}`); for (let [i, test] of tests.entries()) {
let [service, type] = test.split('.');
if (!(service in suite)) throw new Error(`Invalid service: ${service}`);
if (!type) { if (!type) {
args.splice(i + 1, 0, ...Object.keys(stage[service]).map(type => `${arg}.${type}`)); tests.splice(i + 1, 0, ...Object.keys(suite[service]).map(type => `${test}.${type}`));
continue; continue;
} }
let {uri, filter = [], expect} = stage[service][type]; let {uri, filter = [], expect} = suite[service][type];
let test_stage_path = join(stage_path, test);
let preargs = ['--no-logo', '--no-header', '--no-bar']; let preargs = ['--no-logo', '--no-header', '--no-bar'];
if (is_gha) preargs.push('--no-auth'); if (is_gha) preargs.push('--no-auth');
let child_args = [...preargs, uri, ...filter.map(f => `--filter=${f}`)]; if (force) preargs.push('--force');
let child_args = [...preargs, ...filter.map(f => `--filter=${f}`)];
let unmetExpectations = new Error('One or more expectations failed'); let unmetExpectations = new Error('One or more expectations failed');
let child_id = randomUUID();
await pRetry(3, async (attempt, lastErr, abort) => { await pRetry(3, async (attempt, lastErr, abort) => {
if (attempt > 1 && lastErr !== unmetExpectations) abort(); if (attempt > 1 && lastErr !== unmetExpectations) abort();
let logFile = await fileMgr({ let logFile = await fileMgr({
filename: `${service}-${type}-${attempt}.log`, filename: `${service}-${type}-${attempt}.log`,
dirname: path.join('freyr-test', child_id), tmpdir: test_stage_path,
keep: true, keep: true,
mode: fs.constants.W_OK, mode: fs.constants.W_OK,
}); });
@ -117,8 +145,15 @@ async function run_tests(stage, args) {
let child, handler; let child, handler;
if (!docker_image) { if (!docker_image) {
child = spawn('node', [path.relative(process.cwd(), path.join(__dirname, '..', 'cli.js')), ...child_args]); child = spawn('node', [
short_path(join(__dirname, '..', 'cli.js')),
...child_args,
'--directory',
short_path(test_stage_path),
uri,
]);
} else { } else {
let child_id = `${test}.${stage_name}`;
let extra_docker_args = process.env['DOCKER_ARGS'] ? process.env['DOCKER_ARGS'].split(' ') : []; let extra_docker_args = process.env['DOCKER_ARGS'] ? process.env['DOCKER_ARGS'].split(' ') : [];
child = spawn('docker', [ child = spawn('docker', [
'run', 'run',
@ -130,8 +165,11 @@ async function run_tests(stage, args) {
child_id, child_id,
'--network', '--network',
'host', 'host',
'--volume',
`${test_stage_path}:/data`,
docker_image, docker_image,
...child_args, ...child_args,
uri,
]); ]);
process.on('SIGINT', (handler = () => (spawn('docker', ['kill', child_id]), process.off('SIGINT', handler)))); process.on('SIGINT', (handler = () => (spawn('docker', ['kill', child_id]), process.off('SIGINT', handler))));
} }
@ -205,69 +243,74 @@ async function run_tests(stage, args) {
} }
let errorCauses = Symbol('ErrorCauses'); let errorCauses = Symbol('ErrorCauses');
let noService = Symbol('noService');
function main() { async function main(args) {
let args = process.argv.slice(2); let suite, test_suite, i;
let stage, test_suite, i; if (~(i = args.indexOf('--suite')) && !(test_suite = args.splice(i, 2)[1])) throw new Error('`--suite` requires a file path');
if ((test_suite = !!~(i = args.indexOf('--suite'))) && !(test_suite = args.splice(i, 2)[1])) suite = JSON.parse(fs.readFileSync(test_suite || join(__dirname, 'default.json')));
throw new Error('`--suite` requires a file path');
try { if (!['-h', '--help'].some(args.includes.bind(args))) {
stage = JSON.parse(fs.readFileSync(test_suite || path.join(__dirname, 'default.json'))); try {
} catch (e) { if (noService !== (await run_tests(suite, args))) return;
(i = ''), (stage = JSON.parse(fs.readFileSync(path.join(__dirname, 'default.json')))); } catch (err) {
console.error("\x1b[33mCouldn't read test suite file\x1b[0m\n", e.message, '\n'); console.error('An error occurred!');
if (errorCauses in err) {
let causes = err[errorCauses];
delete err[errorCauses];
console.error('', err);
for (let cause of causes) console.error('', cause);
} else console.error('', err);
process.exit(1);
}
} }
if (!args.length || i === '' || args.includes('--help') || args.includes('-h')) { console.log('freyr-test');
console.log('freyr-test'); console.log('----------');
console.log('----------'); console.log('Usage: freyr-test [options] [<SERVICE>[.<TYPE>]...]');
console.log('Usage: freyr-test [options] [<SERVICE>[.<TYPE>]...]'); console.log();
console.log(); console.log('Utility for testing the Freyr CLI');
console.log('Utility for testing the Freyr CLI'); console.log();
console.log(); console.log('Options:');
console.log('Options:'); console.log();
console.log(); console.log(` SERVICE ${Object.keys(suite).join(' / ')}`);
console.log(` SERVICE ${Object.keys(stage).join(' / ')}`); console.log(` TYPE ${[...new Set(Object.values(suite).flatMap(s => Object.keys(s)))].join(' / ')}`);
console.log(` TYPE ${[...new Set(Object.values(stage).flatMap(s => Object.keys(s)))].join(' / ')}`); console.log();
console.log(); console.log(' --all run all tests');
console.log(' --all run all tests'); console.log(' --suite <SUITE> use a specific test suite (json)');
console.log(' --suite <SUITE> use a specific test suite (json)'); console.log(' --docker <IMAGE> run tests in a docker container');
console.log(' --docker <IMAGE> run tests in a docker container'); console.log(' --name <NAME> name for this test run (defaults to a random hex string)');
console.log(' --help show this help message'); console.log(` --stage <PATH> directory to stage this test (default: ${default_stage})`);
console.log(); console.log(' --force force overwriting of existing staged files');
console.log('Enviroment Variables:'); console.log(' --clean (when --force is used) clean existing stage before reusing it');
console.log(); console.log(' --help show this help message');
console.log(' DOCKER_ARGS arguments to pass to `docker run`'); console.log();
console.log(); console.log('Enviroment Variables:');
console.log('Example:'); console.log();
console.log(); console.log(' DOCKER_ARGS arguments to pass to `docker run`');
console.log(' $ freyr-test --all'); console.log();
console.log(' runs all tests'); console.log('Example:');
console.log(); console.log();
console.log(' $ freyr-test spotify'); console.log(' $ freyr-test --all');
console.log(' runs all Spotify tests'); console.log(' runs all tests');
console.log(); console.log();
console.log(' $ freyr-test apple_music.album'); console.log(' $ freyr-test spotify');
console.log(' tests downloading an Apple Music album'); console.log(' runs all Spotify tests');
console.log(); console.log();
console.log(' $ freyr-test spotify.track deezer.artist'); console.log(' $ freyr-test apple_music.album');
console.log(' tests downloading a Spotify track and Deezer artist'); console.log(' tests downloading an Apple Music album');
return; console.log();
} console.log(' $ freyr-test spotify.track deezer.artist');
console.log(' tests downloading a Spotify track and Deezer artist');
run_tests(stage, args).catch(err => { console.log();
console.error('An error occurred!'); console.log(' $ freyr-test spotify.track --stage ./stage --name test-run');
if (errorCauses in err) { console.log(' downloads the Spotify test track in ./stage/test-run/spotify.track with logs');
let causes = err[errorCauses];
delete err[errorCauses];
console.error('', err);
for (let cause of causes) console.error('', cause);
} else console.error('', err);
process.exit(1);
});
} }
main(); function _start() {
main(process.argv.slice(2));
}
_start();