#!/usr/bin/env python3 """Extensible script to run one or more Python Linters across a subset of files in parallel.""" import argparse import logging import os import sys from typing import Dict, List import structlog mongo_dir = os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))) # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: sys.path.append(mongo_dir) # pylint: disable=wrong-import-position from buildscripts.linter.filediff import gather_changed_files_for_lint from buildscripts.linter import base from buildscripts.linter import git from buildscripts.linter import mypy from buildscripts.linter import parallel from buildscripts.linter import pydocstyle from buildscripts.linter import runner from buildscripts.linter import yapf # pylint: enable=wrong-import-position # List of supported linters _LINTERS = [ yapf.YapfLinter(), pydocstyle.PyDocstyleLinter(), mypy.MypyLinter(), ] # List of supported SCons linters _SCONS_LINTERS: List[base.LinterBase] = [ yapf.YapfLinter(), ] def get_py_linter(linter_filter): # type: (str) -> List[base.LinterBase] """ Get a list of linters to use. 'all' or None - select all linters 'scons' - get all scons linters 'a,b,c' - a comma delimited list is describes a list of linters to choose """ if linter_filter is None or linter_filter == "all": return _LINTERS if linter_filter == "scons": return _SCONS_LINTERS linter_list = linter_filter.split(",") linter_candidates = [linter for linter in _LINTERS if linter.cmd_name in linter_list] if not linter_candidates: raise ValueError("No linters found for filter '%s'" % (linter_filter)) return linter_candidates def is_interesting_file(file_name): # type: (str) -> bool """Return true if this file should be checked.""" file_denylist = [] # type: List[str] directory_denylist = ["src/third_party"] if file_name in file_denylist or file_name.startswith(tuple(directory_denylist)): return False directory_list = ["buildscripts", "pytests"] return file_name.endswith(".py") and file_name.startswith(tuple(directory_list)) def is_scons_file(file_name): # type: (str) -> bool """Return true if this file is related to SCons.""" file_denylist = [] # type: List[str] directory_denylist = ["site_scons/third_party"] if file_name in file_denylist or file_name.startswith(tuple(directory_denylist)): return False return (file_name.endswith("SConscript") and file_name.startswith("src")) or \ (file_name.endswith(".py") and file_name.startswith("site_scons")) or \ file_name == "SConstruct" def _lint_files(linters: str, config_dict: Dict[str, str], file_names: List[str], fix_command: str = "fix"): """Lint a list of files with clang-format.""" linter_list = get_py_linter(linters) lint_runner = runner.LintRunner() linter_instances = runner.find_linters(linter_list, config_dict) if not linter_instances: sys.exit(1) failed_lint = False for linter in linter_instances: run_fix = lambda param1: lint_runner.run_lint(linter, param1, mongo_dir, fix_command) # pylint: disable=cell-var-from-loop lint_clean = parallel.parallel_process([os.path.abspath(f) for f in file_names], run_fix) if not lint_clean: failed_lint = True if failed_lint: print("ERROR: Code Style does not match coding style") sys.exit(1) def lint_patch(linters, config_dict, file_name): # type: (str, Dict[str, str], List[str]) -> None """Lint patch command entry point.""" file_names = git.get_files_to_check_from_patch(file_name, is_interesting_file) # Patch may have files that we do not want to check which is fine if file_names: _lint_files(linters, config_dict, file_names) def lint_git_diff(linters, config_dict, _): # type: (str, Dict[str, str], List[str]) -> None """Lint git diff command entry point.""" file_names = gather_changed_files_for_lint(is_interesting_file) # Patch may have files that we do not want to check which is fine if file_names: _lint_files(linters, config_dict, file_names) def lint(linters, config_dict, file_names): # type: (str, Dict[str, str], List[str]) -> None """Lint files command entry point.""" all_file_names = git.get_files_to_check(file_names, is_interesting_file) _lint_files(linters, config_dict, all_file_names) def lint_all(linters, config_dict, file_names): # type: (str, Dict[str, str], List[str]) -> None # pylint: disable=unused-argument """Lint files command entry point based on working tree.""" all_file_names = git.get_files_to_check_working_tree(is_interesting_file) _lint_files(linters, config_dict, all_file_names) def lint_scons(linters, config_dict, file_names): # type: (str, Dict[str, str], List[str]) -> None """Lint SCons files command entry point.""" scons_file_names = git.get_files_to_check(file_names, is_scons_file) _lint_files(linters, config_dict, scons_file_names, "fix-scons") def _fix_files(linters, config_dict, file_names): # type: (str, Dict[str, str], List[str]) -> None """Fix a list of files with linters if possible.""" linter_list = get_py_linter(linters) # Get a list of linters which return a valid command for get_fix_cmd() fix_list = [fixer for fixer in linter_list if fixer.get_fix_cmd_args("ignore")] if not fix_list: raise ValueError("Cannot find any linters '%s' that support fixing." % (linters)) lint_runner = runner.LintRunner() linter_instances = runner.find_linters(fix_list, config_dict) if not linter_instances: sys.exit(1) for linter in linter_instances: run_linter = lambda param1: lint_runner.run(linter.cmd_path + linter.linter. # pylint: disable=cell-var-from-loop get_fix_cmd_args(param1)) lint_clean = parallel.parallel_process([os.path.abspath(f) for f in file_names], run_linter) if not lint_clean: print("ERROR: Code Style does not match coding style") sys.exit(1) def fix_func(linters, config_dict, file_names): # type: (str, Dict[str, str], List[str]) -> None """Fix files command entry point.""" all_file_names = git.get_files_to_check(file_names, is_interesting_file) _fix_files(linters, config_dict, all_file_names) def fix_scons_func(linters, config_dict, file_names): # type: (str, Dict[str, str], List[str]) -> None """Fix SCons files command entry point.""" scons_file_names = git.get_files_to_check(file_names, is_scons_file) _fix_files(linters, config_dict, scons_file_names) def main(): # type: () -> None """Execute Main entry point.""" parser = argparse.ArgumentParser( description='PyLinter frontend; see more details at https://wiki.corp.mongodb.com/x/1vP5BQ') linters = get_py_linter(None) dest_prefix = "linter_" for linter1 in linters: msg = 'Path to linter %s' % (linter1.cmd_name) parser.add_argument('--' + linter1.cmd_name, type=str, help=msg, dest=dest_prefix + linter1.cmd_name) parser.add_argument('--linters', type=str, help="Comma separated list of filters to use, defaults to 'all'", default="all") parser.add_argument('-v', "--verbose", action='store_true', help="Enable verbose logging") sub = parser.add_subparsers(title="Linter subcommands", help="sub-command help") parser_lint = sub.add_parser('lint', help='Lint only Git files') parser_lint.add_argument("file_names", nargs="*", help="Globs of files to check") parser_lint.set_defaults(func=lint) parser_lint_all = sub.add_parser('lint-all', help='Lint All files') parser_lint_all.add_argument("file_names", nargs="*", help="Globs of files to check") parser_lint_all.set_defaults(func=lint_all) parser_lint_patch = sub.add_parser('lint-patch', help='Lint the files in a patch') parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check") parser_lint_patch.set_defaults(func=lint_patch) parser_lint_patch = sub.add_parser('lint-git-diff', help='Lint the files since the last git commit') parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check") parser_lint_patch.set_defaults(func=lint_git_diff) parser_fix = sub.add_parser('fix', help='Fix files if possible') parser_fix.add_argument("file_names", nargs="*", help="Globs of files to check") parser_fix.set_defaults(func=fix_func) parser_lint = sub.add_parser('lint-scons', help='Lint only SCons files') parser_lint.add_argument("file_names", nargs="*", help="Globs of files to check") parser_lint.set_defaults(func=lint_scons, linters="scons") parser_fix = sub.add_parser('fix-scons', help='Fix SCons related files if possible') parser_fix.add_argument("file_names", nargs="*", help="Globs of files to check") parser_fix.set_defaults(func=fix_scons_func, linters="scons") # No args given? Fall back to usage screen: if len(sys.argv) == 1: parser.print_help() return args = parser.parse_args() # Create a dictionary of linter locations if the user needs to override the location of a # linter. This is common for mypy on Windows for instance. config_dict = {} for key in args.__dict__: if key.startswith("linter_"): name = key.replace(dest_prefix, "") config_dict[name] = args.__dict__[key] if args.verbose: logging.basicConfig(level=logging.DEBUG) structlog.configure(logger_factory=structlog.stdlib.LoggerFactory()) args.func(args.linters, config_dict, args.file_names) if __name__ == "__main__": main()