mongo/buildscripts/pylinters.py

#!/usr/bin/env python3
"""Extensible script to run one or more Python Linters across a subset of files in parallel."""

import argparse
import logging
import os
import sys
from typing import Dict, List

import structlog

mongo_dir = os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
    sys.path.append(mongo_dir)

# pylint: disable=wrong-import-position
from buildscripts.linter.filediff import gather_changed_files_for_lint
from buildscripts.linter import base
from buildscripts.linter import git
from buildscripts.linter import mypy
from buildscripts.linter import parallel
from buildscripts.linter import pydocstyle
from buildscripts.linter import runner
from buildscripts.linter import yapf
# pylint: enable=wrong-import-position

# List of supported linters
_LINTERS = [
    yapf.YapfLinter(),
    pydocstyle.PyDocstyleLinter(),
    mypy.MypyLinter(),
]

# List of supported SCons linters
_SCONS_LINTERS: List[base.LinterBase] = [
    yapf.YapfLinter(),
]


def get_py_linter(linter_filter):
    # type: (str) -> List[base.LinterBase]
    """
    Get a list of linters to use.

    'all' or None - select all linters
    'scons' - get all scons linters
    'a,b,c' - a comma delimited list is describes a list of linters to choose
    """
    if linter_filter is None or linter_filter == "all":
        return _LINTERS

    if linter_filter == "scons":
        return _SCONS_LINTERS

    linter_list = linter_filter.split(",")

    linter_candidates = [linter for linter in _LINTERS if linter.cmd_name in linter_list]

    if not linter_candidates:
        raise ValueError("No linters found for filter '%s'" % (linter_filter))

    return linter_candidates


def is_interesting_file(file_name):
    # type: (str) -> bool
    """Return true if this file should be checked."""
    file_denylist = []  # type: List[str]
    directory_denylist = ["src/third_party"]
    if file_name in file_denylist or file_name.startswith(tuple(directory_denylist)):
        return False
    directory_list = ["buildscripts", "pytests"]
    return file_name.endswith(".py") and file_name.startswith(tuple(directory_list))


def is_scons_file(file_name):
    # type: (str) -> bool
    """Return true if this file is related to SCons."""
    file_denylist = []  # type: List[str]
    directory_denylist = ["site_scons/third_party"]
    if file_name in file_denylist or file_name.startswith(tuple(directory_denylist)):
        return False
    return (file_name.endswith("SConscript") and file_name.startswith("src")) or \
            (file_name.endswith(".py") and file_name.startswith("site_scons")) or \
            file_name == "SConstruct"


def _lint_files(linters: str, config_dict: Dict[str, str], file_names: List[str],
                fix_command: str = "fix"):
    """Lint a list of files with clang-format."""
    linter_list = get_py_linter(linters)

    lint_runner = runner.LintRunner()

    linter_instances = runner.find_linters(linter_list, config_dict)
    if not linter_instances:
        sys.exit(1)

    failed_lint = False

    for linter in linter_instances:
        run_fix = lambda param1: lint_runner.run_lint(linter, param1, mongo_dir, fix_command)  # pylint: disable=cell-var-from-loop
        lint_clean = parallel.parallel_process([os.path.abspath(f) for f in file_names], run_fix)

        if not lint_clean:
            failed_lint = True

    if failed_lint:
        print("ERROR: Code Style does not match coding style")
        sys.exit(1)


def lint_patch(linters, config_dict, file_name):
    # type: (str, Dict[str, str], List[str]) -> None
    """Lint patch command entry point."""
    file_names = git.get_files_to_check_from_patch(file_name, is_interesting_file)

    # Patch may have files that we do not want to check which is fine
    if file_names:
        _lint_files(linters, config_dict, file_names)


def lint_git_diff(linters, config_dict, _):
    # type: (str, Dict[str, str], List[str]) -> None
    """Lint git diff command entry point."""
    file_names = gather_changed_files_for_lint(is_interesting_file)

    # Patch may have files that we do not want to check which is fine
    if file_names:
        _lint_files(linters, config_dict, file_names)


def lint(linters, config_dict, file_names):
    # type: (str, Dict[str, str], List[str]) -> None
    """Lint files command entry point."""
    all_file_names = git.get_files_to_check(file_names, is_interesting_file)

    _lint_files(linters, config_dict, all_file_names)


def lint_all(linters, config_dict, file_names):
    # type: (str, Dict[str, str], List[str]) -> None
    # pylint: disable=unused-argument
    """Lint files command entry point based on working tree."""
    all_file_names = git.get_files_to_check_working_tree(is_interesting_file)

    _lint_files(linters, config_dict, all_file_names)


def lint_scons(linters, config_dict, file_names):
    # type: (str, Dict[str, str], List[str]) -> None
    """Lint SCons files command entry point."""
    scons_file_names = git.get_files_to_check(file_names, is_scons_file)

    _lint_files(linters, config_dict, scons_file_names, "fix-scons")


def _fix_files(linters, config_dict, file_names):
    # type: (str, Dict[str, str], List[str]) -> None
    """Fix a list of files with linters if possible."""
    linter_list = get_py_linter(linters)

    # Get a list of linters which return a valid command for get_fix_cmd()
    fix_list = [fixer for fixer in linter_list if fixer.get_fix_cmd_args("ignore")]

    if not fix_list:
        raise ValueError("Cannot find any linters '%s' that support fixing." % (linters))

    lint_runner = runner.LintRunner()

    linter_instances = runner.find_linters(fix_list, config_dict)
    if not linter_instances:
        sys.exit(1)

    for linter in linter_instances:
        run_linter = lambda param1: lint_runner.run(linter.cmd_path + linter.linter.  # pylint: disable=cell-var-from-loop
                                                    get_fix_cmd_args(param1))

        lint_clean = parallel.parallel_process([os.path.abspath(f) for f in file_names], run_linter)

        if not lint_clean:
            print("ERROR: Code Style does not match coding style")
            sys.exit(1)


def fix_func(linters, config_dict, file_names):
    # type: (str, Dict[str, str], List[str]) -> None
    """Fix files command entry point."""
    all_file_names = git.get_files_to_check(file_names, is_interesting_file)

    _fix_files(linters, config_dict, all_file_names)


def fix_scons_func(linters, config_dict, file_names):
    # type: (str, Dict[str, str], List[str]) -> None
    """Fix SCons files command entry point."""
    scons_file_names = git.get_files_to_check(file_names, is_scons_file)

    _fix_files(linters, config_dict, scons_file_names)


def main():
    # type: () -> None
    """Execute Main entry point."""

    parser = argparse.ArgumentParser(
        description='PyLinter frontend; see more details at https://wiki.corp.mongodb.com/x/1vP5BQ')

    linters = get_py_linter(None)

    dest_prefix = "linter_"
    for linter1 in linters:
        msg = 'Path to linter %s' % (linter1.cmd_name)
        parser.add_argument('--' + linter1.cmd_name, type=str, help=msg,
                            dest=dest_prefix + linter1.cmd_name)

    parser.add_argument('--linters', type=str,
                        help="Comma separated list of filters to use, defaults to 'all'",
                        default="all")

    parser.add_argument('-v', "--verbose", action='store_true', help="Enable verbose logging")

    sub = parser.add_subparsers(title="Linter subcommands", help="sub-command help")

    parser_lint = sub.add_parser('lint', help='Lint only Git files')
    parser_lint.add_argument("file_names", nargs="*", help="Globs of files to check")
    parser_lint.set_defaults(func=lint)

    parser_lint_all = sub.add_parser('lint-all', help='Lint All files')
    parser_lint_all.add_argument("file_names", nargs="*", help="Globs of files to check")
    parser_lint_all.set_defaults(func=lint_all)

    parser_lint_patch = sub.add_parser('lint-patch', help='Lint the files in a patch')
    parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check")
    parser_lint_patch.set_defaults(func=lint_patch)

    parser_lint_patch = sub.add_parser('lint-git-diff',
                                       help='Lint the files since the last git commit')
    parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check")
    parser_lint_patch.set_defaults(func=lint_git_diff)

    parser_fix = sub.add_parser('fix', help='Fix files if possible')
    parser_fix.add_argument("file_names", nargs="*", help="Globs of files to check")
    parser_fix.set_defaults(func=fix_func)

    parser_lint = sub.add_parser('lint-scons', help='Lint only SCons files')
    parser_lint.add_argument("file_names", nargs="*", help="Globs of files to check")
    parser_lint.set_defaults(func=lint_scons, linters="scons")

    parser_fix = sub.add_parser('fix-scons', help='Fix SCons related files if possible')
    parser_fix.add_argument("file_names", nargs="*", help="Globs of files to check")
    parser_fix.set_defaults(func=fix_scons_func, linters="scons")

    # No args given? Fall back to usage screen:
    if len(sys.argv) == 1:
        parser.print_help()
        return
    args = parser.parse_args()

    # Create a dictionary of linter locations if the user needs to override the location of a
    # linter. This is common for mypy on Windows for instance.
    config_dict = {}
    for key in args.__dict__:
        if key.startswith("linter_"):
            name = key.replace(dest_prefix, "")
            config_dict[name] = args.__dict__[key]

    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    structlog.configure(logger_factory=structlog.stdlib.LoggerFactory())

    args.func(args.linters, config_dict, args.file_names)


if __name__ == "__main__":
    main()