diff --git a/bazel/format/rules_lint_format_wrapper.py b/bazel/format/rules_lint_format_wrapper.py index d66f908b1e7..630b38183f8 100644 --- a/bazel/format/rules_lint_format_wrapper.py +++ b/bazel/format/rules_lint_format_wrapper.py @@ -184,10 +184,11 @@ def main() -> int: files_to_format = "all" if not args.all: + max_distance = 100 distance = _git_distance([f"{args.origin_branch}..HEAD"]) - if distance > 100: + if distance > max_distance: print( - f"The number of commits between current branch and origin branch ({args.origin_branch}) is too large: {distance} commits" + f"The number of commits between current branch and origin branch ({args.origin_branch}) is too large: {distance} commits (> {max_distance} commits)." ) print("WARNING!!! Defaulting to formatting all files, this may take a while.") print( diff --git a/bazel/wrapper_hook/lint.py b/bazel/wrapper_hook/lint.py index 6c0d661c05c..f230d4f3f0c 100644 --- a/bazel/wrapper_hook/lint.py +++ b/bazel/wrapper_hook/lint.py @@ -126,6 +126,59 @@ def list_files_without_targets( return True +def _git_distance(args: list) -> int: + command = ["git", "rev-list", "--count"] + args + try: + result = subprocess.run(command, capture_output=True, text=True, check=True) + except subprocess.CalledProcessError as e: + print(f"Error running git command: {' '.join(command)}") + print(f"stderr: {e.stderr.strip()}") + print(f"stdout: {e.stdout.strip()}") + raise + return int(result.stdout.strip()) + + +def _get_merge_base(args: list) -> str: + command = ["git", "merge-base"] + args + result = subprocess.run(command, capture_output=True, text=True, check=True) + return result.stdout.strip() + + +def _git_diff(args: list) -> str: + command = ["git", "diff"] + args + result = subprocess.run(command, capture_output=True, text=True, check=True) + return result.stdout.strip() + os.linesep + + +def _git_unstaged_files() -> str: + command = ["git", "ls-files", "--others", "--exclude-standard"] + result = subprocess.run(command, capture_output=True, text=True, check=True) + return result.stdout.strip() + os.linesep + + +def _get_files_changed_since_fork_point(origin_branch: str = "origin/master") -> List[str]: + """Query git to get a list of files in the repo from a diff.""" + # There are 3 diffs we run: + # 1. List of commits between origin/master and HEAD of current branch + # 2. Cached/Staged files (--cached) + # 3. Working Tree files git tracks + + fork_point = _get_merge_base(["HEAD", origin_branch]) + + diff_files = _git_diff(["--name-only", f"{fork_point}..HEAD"]) + diff_files += _git_diff(["--name-only", "--cached"]) + diff_files += _git_diff(["--name-only"]) + diff_files += _git_unstaged_files() + + file_set = { + os.path.normpath(os.path.join(os.curdir, line.rstrip())) + for line in diff_files.splitlines() + if line + } + + return list(file_set) + + def run_rules_lint(bazel_bin: str, args: List[str]) -> bool: if platform.system() == "Windows": print("eslint not supported on windows") @@ -148,9 +201,50 @@ def run_rules_lint(bazel_bin: str, args: List[str]) -> bool: ): return False - # Default to linting everything if no path was passed in + lint_all = False if len([arg for arg in args if not arg.startswith("--")]) == 0: - args = ["//..."] + args + origin_branch = "origin/master" + for arg in args: + if arg.startswith("--origin-branch="): + origin_branch = arg.split("=")[1] + args.remove(arg) + + max_distance = 100 + distance = _git_distance([f"{origin_branch}..HEAD"]) + if distance > max_distance: + print( + f"The number of commits between current branch and origin branch ({origin_branch}) is too large: {distance} commits (> {max_distance} commits)." + ) + print("WARNING!!! Defaulting to formatting all files, this may take a while.") + print( + "Please update your local branch with the latest changes from origin, or use `bazel run lint -- --origin-branch=other_branch` to select a different origin branch" + ) + lint_all = True + else: + files_to_format = [ + file + for file in _get_files_changed_since_fork_point(origin_branch) + if file.endswith((".cpp", ".c", ".h", ".py", ".js", ".mjs", ".json")) + ] + args = files_to_format + args + ["--compile_one_dependency"] + + if lint_all or "sbom.json" in args: + subprocess.run([bazel_bin, "run", "//buildscripts:sbom_linter"], check=True) + + if lint_all or any(file.endswith(".h") or file.endswith(".cpp") for file in args): + subprocess.run( + [bazel_bin, "run", "//buildscripts:quickmongolint", "--", "lint"], check=True + ) + + if lint_all or any( + file.endswith(".cpp") + or file.endswith(".c") + or file.endswith(".h") + or file.endswith(".py") + or file.endswith(".idl") + for file in args + ): + subprocess.run([bazel_bin, "run", "//buildscripts:errorcodes", "--", "--quiet"], check=True) fix = "" with tempfile.NamedTemporaryFile(delete=False) as buildevents: diff --git a/buildscripts/BUILD.bazel b/buildscripts/BUILD.bazel index 7b62fbf0e14..d9a044c506c 100644 --- a/buildscripts/BUILD.bazel +++ b/buildscripts/BUILD.bazel @@ -153,6 +153,29 @@ py_binary( ], ) +py_binary( + name = "quickmongolint", + srcs = [ + "quickmongolint.py", + ], + visibility = ["//visibility:public"], + deps = [ + "//buildscripts/linter", + ], +) + +py_binary( + name = "errorcodes", + srcs = ["errorcodes.py"], + visibility = ["//visibility:public"], + deps = [ + dependency( + "regex", + group = "compile", + ), + ], +) + bin.eslint_binary( name = "eslint_binary", # Allow the binary to be run outside bazel diff --git a/buildscripts/errorcodes.py b/buildscripts/errorcodes.py index c32afd12477..8ee2f1ee98a 100755 --- a/buildscripts/errorcodes.py +++ b/buildscripts/errorcodes.py @@ -290,6 +290,9 @@ def coerce_to_number(ticket_value): def main(): """Validate error codes.""" + + os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", ".")) + parser = OptionParser(description=__doc__.strip()) parser.add_option( "--fix", diff --git a/buildscripts/quickmongolint.py b/buildscripts/quickmongolint.py index d3254a20320..de3e78509eb 100755 --- a/buildscripts/quickmongolint.py +++ b/buildscripts/quickmongolint.py @@ -79,6 +79,8 @@ def lint_my(origin_branch: List[str]) -> None: def main() -> None: """Execute Main entry point.""" + os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", ".")) + parser = argparse.ArgumentParser(description="Quick C++ Lint frontend.") parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose logging") diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index e7c4c11923e..fba4acca3d2 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -737,36 +737,6 @@ tasks: - "buildscripts/quickmongolint.py" - "lint" - - name: lint_errorcodes - tags: - [ - "assigned_to_jira_team_devprod_build", - "development_critical_single_variant", - "lint", - ] - commands: - - command: timeout.update - params: - # 20 mins - exec_timeout_secs: 1200 - - func: "f_expansions_write" - - command: manifest.load - - func: "git get project and add git tag" - - func: "f_expansions_write" - - func: "kill processes" - - func: "cleanup environment" - - func: "set up venv" - - func: "upload pip requirements" - - command: subprocess.exec - type: test - params: - binary: bash - args: - - "./src/evergreen/run_python_script_with_report.sh" - - "lint-errorcodes" - - "buildscripts/errorcodes.py" - - "--quiet" - - name: bazel_run_lint tags: [ @@ -794,7 +764,7 @@ tasks: - func: "bazel run" vars: target: >- - lint + lint //... # Check that the mutational fuzzer can parse all JS filess. - name: lint_fuzzer_sanity_all @@ -884,34 +854,6 @@ tasks: - "lint-poetry-lock" - "evergreen/functions/poetry_lock_check.py" - # TODO: DEVPROD-9047 Enable after the issue with getting third party file is resolved - - name: lint_sbom - tags: - [ - "assigned_to_jira_team_devprod_correctness", - "development_critical_single_variant", - "lint", - ] - commands: - - command: timeout.update - params: - # 20 mins - exec_timeout_secs: 1200 - - func: "f_expansions_write" - - command: manifest.load - - func: "git get project and add git tag" - - func: "f_expansions_write" - - func: "kill processes" - - func: "cleanup environment" - - func: "set up venv" - - command: subprocess.exec - params: - binary: bash - args: - - "src/evergreen/run_python_script_with_report.sh" - - "lint-sbom" - - "buildscripts/sbom_linter.py" - - name: lint_pyright tags: [