From f38ed57ed59f0e78ebb9e80b7e744af32f3b4bf5 Mon Sep 17 00:00:00 2001 From: Tyler Wilding Date: Fri, 4 Sep 2020 20:40:07 -0400 Subject: [PATCH] Update workflow --- .github/workflows/linux-workflow.yaml | 27 -- .github/workflows/workflow.yaml | 82 +++++ CMakeSettings.json | 42 +-- third-party/run-clang-tidy/run-clang-tidy.py | 314 +++++++++++++++++++ 4 files changed, 417 insertions(+), 48 deletions(-) delete mode 100644 .github/workflows/linux-workflow.yaml create mode 100644 .github/workflows/workflow.yaml create mode 100644 third-party/run-clang-tidy/run-clang-tidy.py diff --git a/.github/workflows/linux-workflow.yaml b/.github/workflows/linux-workflow.yaml deleted file mode 100644 index fbb17ac238..0000000000 --- a/.github/workflows/linux-workflow.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Linux Workflow -on: [push, pull_request] - -jobs: - build: - name: Build and Test Project (Linux) - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v2 - - name: Get Package Dependencies - run: sudo apt install gcc make cmake build-essential g++ nasm clang-format - - name: Initialize Submodules - run: git submodule update --init --recursive - - name: Build Project with CMake - run: | - mkdir build - cd build - cmake .. - make -j - - name: Test Project with gTest - run: ./test.sh - timeout-minutes: 5 - - name: Check Clang-Formatting - run: | - chmod +x ./third-party/run-clang-format/run-clang-format.py - ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test --color always diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml new file mode 100644 index 0000000000..9ae6baf58b --- /dev/null +++ b/.github/workflows/workflow.yaml @@ -0,0 +1,82 @@ +name: Build +on: [push, pull_request] + +jobs: + common: + # TODO - Start using clang-tidy - requires a bit more setup though - http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html + name: Linting & Formatting + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + with: + submodules: true + - name: Get Package Dependencies + run: sudo apt install clang-format clang-tidy + - name: Check Clang-Formatting + run: | + chmod +x ./third-party/run-clang-format/run-clang-format.py + ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test --color always + build-linux: + name: (Linux) Build & Test + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./ + steps: + - name: Checkout Repository + uses: actions/checkout@v1 + with: + submodules: true + - name: Get Package Dependencies + run: sudo apt install build-essential cmake gcc g++ make nasm + - name: CMake Generation + run: | + cmake --version + cmake -B build + - name: Build Project + run: | + cd build + make -j2 + - name: Run Tests + timeout-minutes: 5 + run: ./test.sh + build-windows: + # Very good resource - https://cristianadam.eu/20191222/using-github-actions-with-c-plus-plus-and-cmake/ + name: (Windows) Build & Test + runs-on: windows-latest + defaults: + run: + shell: powershell + working-directory: ./ + steps: + - name: Checkout Repository + uses: actions/checkout@v1 + with: + submodules: true + # https://github.com/ilammy/setup-nasm + - uses: ilammy/setup-nasm@v1 + # TODO - we need to make a release configuration! (i think linux needs one too) + - name: CMake Generation + shell: cmd # ideally id like everything to be powershell but running this bat file is finicky + run: | + call "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Auxiliary/Build/vcvars64.bat" + cmake --version + cmake -B build -DCMAKE_BUILD_TYPE=Debug -G "NMake Makefiles" . + - name: Build Project + working-directory: ./build + shell: cmd + run: | + call "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Auxiliary/Build/vcvars64.bat" + nmake + - name: Run Tests + timeout-minutes: 5 + run: | + ls + ls ./build + ls ./out + ls ./test + $env:NEXT_DIR = $pwd + $env:FAKE_ISO_PATH = "/game/fake_iso.txt" + & './build/bin/goalc-test.exe' diff --git a/CMakeSettings.json b/CMakeSettings.json index 298f497edd..fcd25ae01f 100644 --- a/CMakeSettings.json +++ b/CMakeSettings.json @@ -1,22 +1,22 @@ { - "configurations": [ - { - "name": "x64-Debug", - "generator": "Ninja", - "configurationType": "Debug", - "inheritEnvironments": [ "msvc_x64_x64" ], - "buildRoot": "${projectDir}\\out\\build\\${name}", - "installRoot": "${projectDir}\\out\\install\\${name}", - "cmakeCommandArgs": "", - "buildCommandArgs": "", - "ctestCommandArgs": "", - "variables": [ - { - "name": "INSTALL_GTEST", - "value": "True", - "type": "BOOL" - } - ] - } - ] -} \ No newline at end of file + "configurations": [ + { + "name": "Debug", + "generator": "Ninja", + "configurationType": "Debug", + "inheritEnvironments": ["msvc_x64_x64"], + "buildRoot": "${projectDir}\\out\\build\\${name}", + "installRoot": "${projectDir}\\out\\install\\${name}", + "cmakeCommandArgs": "", + "buildCommandArgs": "", + "ctestCommandArgs": "", + "variables": [ + { + "name": "INSTALL_GTEST", + "value": "True", + "type": "BOOL" + } + ] + } + ] +} diff --git a/third-party/run-clang-tidy/run-clang-tidy.py b/third-party/run-clang-tidy/run-clang-tidy.py new file mode 100644 index 0000000000..ae2cf92b93 --- /dev/null +++ b/third-party/run-clang-tidy/run-clang-tidy.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python +# +#===- run-clang-tidy.py - Parallel clang-tidy runner ---------*- python -*--===# +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +#===------------------------------------------------------------------------===# +# FIXME: Integrate with clang-tidy-diff.py +# NOTE: Original in: https://github.com/llvm-mirror/clang-tools-extra/blob/master/clang-tidy/tool/run-clang-tidy.py + +""" +Parallel clang-tidy runner +========================== + +Runs clang-tidy over all files in a compilation database. Requires clang-tidy +and clang-apply-replacements in $PATH. + +Example invocations. +- Run clang-tidy on all files in the current working directory with a default + set of checks and show warnings in the cpp files and all project headers. + run-clang-tidy.py $PWD + +- Fix all header guards. + run-clang-tidy.py -fix -checks=-*,llvm-header-guard + +- Fix all header guards included from clang-tidy and header guards + for clang-tidy headers. + run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \ + -header-filter=extra/clang-tidy + +Compilation database setup: +http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html +""" + +from __future__ import print_function + +import argparse +import glob +import json +import multiprocessing +import os +import re +import shutil +import subprocess +import sys +import tempfile +import threading +import traceback +import yaml + +is_py2 = sys.version[0] == '2' + +if is_py2: + import Queue as queue +else: + import queue as queue + +def find_compilation_database(path): + """Adjusts the directory until a compilation database is found.""" + result = './' + while not os.path.isfile(os.path.join(result, path)): + if os.path.realpath(result) == '/': + print('Error: could not find compilation database.') + sys.exit(1) + result += '../' + return os.path.realpath(result) + + +def make_absolute(f, directory): + if os.path.isabs(f): + return f + return os.path.normpath(os.path.join(directory, f)) + + +def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path, + header_filter, extra_arg, extra_arg_before, quiet, + config): + """Gets a command line for clang-tidy.""" + start = [clang_tidy_binary] + if header_filter is not None: + start.append('-header-filter=' + header_filter) + else: + # Show warnings in all in-project headers by default. + start.append('-header-filter=^' + build_path + '/.*') + if checks: + start.append('-checks=' + checks) + if tmpdir is not None: + start.append('-export-fixes') + # Get a temporary file. We immediately close the handle so clang-tidy can + # overwrite it. + (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir) + os.close(handle) + start.append(name) + for arg in extra_arg: + start.append('-extra-arg=%s' % arg) + for arg in extra_arg_before: + start.append('-extra-arg-before=%s' % arg) + start.append('-p=' + build_path) + if quiet: + start.append('-quiet') + if config: + start.append('-config=' + config) + start.append(f) + return start + + +def merge_replacement_files(tmpdir, mergefile): + """Merge all replacement files in a directory into a single file""" + # The fixes suggested by clang-tidy >= 4.0.0 are given under + # the top level key 'Diagnostics' in the output yaml files + mergekey="Diagnostics" + merged=[] + for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')): + content = yaml.safe_load(open(replacefile, 'r')) + if not content: + continue # Skip empty files. + merged.extend(content.get(mergekey, [])) + + if merged: + # MainSourceFile: The key is required by the definition inside + # include/clang/Tooling/ReplacementsYaml.h, but the value + # is actually never used inside clang-apply-replacements, + # so we set it to '' here. + output = { 'MainSourceFile': '', mergekey: merged } + with open(mergefile, 'w') as out: + yaml.safe_dump(output, out) + else: + # Empty the file: + open(mergefile, 'w').close() + + +def check_clang_apply_replacements_binary(args): + """Checks if invoking supplied clang-apply-replacements binary works.""" + try: + subprocess.check_call([args.clang_apply_replacements_binary, '--version']) + except: + print('Unable to run clang-apply-replacements. Is clang-apply-replacements ' + 'binary correctly specified?', file=sys.stderr) + traceback.print_exc() + sys.exit(1) + + +def apply_fixes(args, tmpdir): + """Calls clang-apply-fixes on a given directory.""" + invocation = [args.clang_apply_replacements_binary] + if args.format: + invocation.append('-format') + if args.style: + invocation.append('-style=' + args.style) + invocation.append(tmpdir) + subprocess.call(invocation) + + +def run_tidy(args, tmpdir, build_path, queue, failed_files): + """Takes filenames out of queue and runs clang-tidy on them.""" + while True: + name = queue.get() + invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks, + tmpdir, build_path, args.header_filter, + args.extra_arg, args.extra_arg_before, + args.quiet, args.config) + sys.stdout.write(' '.join(invocation) + '\n') + return_code = subprocess.call(invocation) + if return_code != 0: + failed_files.append(name) + queue.task_done() + + +def main(): + parser = argparse.ArgumentParser(description='Runs clang-tidy over all files ' + 'in a compilation database. Requires ' + 'clang-tidy and clang-apply-replacements in ' + '$PATH.') + parser.add_argument('-clang-tidy-binary', metavar='PATH', + default='clang-tidy', + help='path to clang-tidy binary') + parser.add_argument('-clang-apply-replacements-binary', metavar='PATH', + default='clang-apply-replacements', + help='path to clang-apply-replacements binary') + parser.add_argument('-checks', default=None, + help='checks filter, when not specified, use clang-tidy ' + 'default') + parser.add_argument('-config', default=None, + help='Specifies a configuration in YAML/JSON format: ' + ' -config="{Checks: \'*\', ' + ' CheckOptions: [{key: x, ' + ' value: y}]}" ' + 'When the value is empty, clang-tidy will ' + 'attempt to find a file named .clang-tidy for ' + 'each source file in its parent directories.') + parser.add_argument('-header-filter', default=None, + help='regular expression matching the names of the ' + 'headers to output diagnostics from. Diagnostics from ' + 'the main file of each translation unit are always ' + 'displayed.') + parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes', + help='Create a yaml file to store suggested fixes in, ' + 'which can be applied with clang-apply-replacements.') + parser.add_argument('-j', type=int, default=0, + help='number of tidy instances to be run in parallel.') + parser.add_argument('files', nargs='*', default=['.*'], + help='files to be processed (regex on path)') + parser.add_argument('-fix', action='store_true', help='apply fix-its') + parser.add_argument('-format', action='store_true', help='Reformat code ' + 'after applying fixes') + parser.add_argument('-style', default='file', help='The style of reformat ' + 'code after applying fixes') + parser.add_argument('-p', dest='build_path', + help='Path used to read a compile command database.') + parser.add_argument('-extra-arg', dest='extra_arg', + action='append', default=[], + help='Additional argument to append to the compiler ' + 'command line.') + parser.add_argument('-extra-arg-before', dest='extra_arg_before', + action='append', default=[], + help='Additional argument to prepend to the compiler ' + 'command line.') + parser.add_argument('-quiet', action='store_true', + help='Run clang-tidy in quiet mode') + args = parser.parse_args() + + db_path = 'compile_commands.json' + + if args.build_path is not None: + build_path = args.build_path + else: + # Find our database + build_path = find_compilation_database(db_path) + + try: + invocation = [args.clang_tidy_binary, '-list-checks'] + invocation.append('-p=' + build_path) + if args.checks: + invocation.append('-checks=' + args.checks) + invocation.append('-') + subprocess.check_call(invocation) + except: + print("Unable to run clang-tidy.", file=sys.stderr) + sys.exit(1) + + # Load the database and extract all files. + database = json.load(open(os.path.join(build_path, db_path))) + files = [make_absolute(entry['file'], entry['directory']) + for entry in database] + + max_task = args.j + if max_task == 0: + max_task = multiprocessing.cpu_count() + + tmpdir = None + if args.fix or args.export_fixes: + check_clang_apply_replacements_binary(args) + tmpdir = tempfile.mkdtemp() + + # Build up a big regexy filter from all command line arguments. + file_name_re = re.compile('|'.join(args.files)) + + return_code = 0 + try: + # Spin up a bunch of tidy-launching threads. + task_queue = queue.Queue(max_task) + # List of files with a non-zero return code. + failed_files = [] + for _ in range(max_task): + t = threading.Thread(target=run_tidy, + args=(args, tmpdir, build_path, task_queue, failed_files)) + t.daemon = True + t.start() + + # Fill the queue with files. + for name in files: + if file_name_re.search(name): + task_queue.put(name) + + # Wait for all threads to be done. + task_queue.join() + if len(failed_files): + return_code = 1 + + except KeyboardInterrupt: + # This is a sad hack. Unfortunately subprocess goes + # bonkers with ctrl-c and we start forking merrily. + print('\nCtrl-C detected, goodbye.') + if tmpdir: + shutil.rmtree(tmpdir) + os.kill(0, 9) + + if args.export_fixes: + print('Writing fixes to ' + args.export_fixes + ' ...') + try: + merge_replacement_files(tmpdir, args.export_fixes) + except: + print('Error exporting fixes.\n', file=sys.stderr) + traceback.print_exc() + return_code=1 + + if args.fix: + print('Applying fixes ...') + try: + apply_fixes(args, tmpdir) + except: + print('Error applying fixes.\n', file=sys.stderr) + traceback.print_exc() + return_code=1 + + if tmpdir: + shutil.rmtree(tmpdir) + sys.exit(return_code) + +if __name__ == '__main__': + main()