From caa07aac4e4580b17095c00d75dcb4516b877ca1 Mon Sep 17 00:00:00 2001 From: Zac Date: Thu, 31 Jul 2025 11:17:09 -0700 Subject: [PATCH] SERVER-108478 Move lint_large_files_check to bazel run lint (#39413) GitOrigin-RevId: e6ff6a58c1b1fe83644e859bdfb4e240c6528112 --- bazel/wrapper_hook/lint.py | 17 ++++++++++- buildscripts/BUILD.bazel | 17 +++++++++++ buildscripts/large_file_check.py | 6 +++- .../tasks/misc_tasks.yml | 29 +------------------ 4 files changed, 39 insertions(+), 30 deletions(-) diff --git a/bazel/wrapper_hook/lint.py b/bazel/wrapper_hook/lint.py index dbb667f2799..03aa150909c 100644 --- a/bazel/wrapper_hook/lint.py +++ b/bazel/wrapper_hook/lint.py @@ -10,6 +10,7 @@ from typing import List REPO_ROOT = pathlib.Path(__file__).parent.parent.parent sys.path.append(str(REPO_ROOT)) +LARGE_FILE_THRESHOLD = 10 * 1024 * 1024 #10MiB def create_build_files_in_new_js_dirs() -> None: base_dirs = ["src/mongo/db/modules/enterprise/jstests", "jstests"] @@ -207,7 +208,7 @@ def get_parsed_args(args): ) parser.add_argument( "--fail-on-validation", - type=bool, + action="store_true", default=False, ) parser.add_argument( @@ -216,6 +217,11 @@ def get_parsed_args(args): default="origin/master", help="Base branch to compare changes against", ) + parser.add_argument( + "--large-files", + action="store_true", + default=False + ) return parser.parse_known_args(args) def run_rules_lint(bazel_bin: str, args: List[str]) -> bool: @@ -282,6 +288,15 @@ def run_rules_lint(bazel_bin: str, args: List[str]) -> bool: if lint_all or any(file.endswith(".yml") for file in files_to_lint): subprocess.run([bazel_bin, "run", "//buildscripts:validate_evg_project_config", "--", f"--evg-project-name={parsed_args.lint_yaml_project}", "--evg-auth-config=.evergreen.yml"], check=True) + if lint_all or parsed_args.large_files: + subprocess.run([bazel_bin, "run", "//buildscripts:large_file_check", "--", "--exclude", "src/third_party/*"], check=True) + else: + # simple check + for file in files_to_lint: + if os.path.getsize(file) > LARGE_FILE_THRESHOLD: + print(f"File {file} exceeds large file threshold of {LARGE_FILE_THRESHOLD}") + return False + # Default to linting everything in rules_lint if no path was passed in. if len([arg for arg in args if not arg.startswith("--")]) == 0: args = ["//..."] + args diff --git a/buildscripts/BUILD.bazel b/buildscripts/BUILD.bazel index b79bcacbc12..8d549633f27 100644 --- a/buildscripts/BUILD.bazel +++ b/buildscripts/BUILD.bazel @@ -306,6 +306,23 @@ py_binary( ], ) +py_binary( + name = "large_file_check", + srcs = ["large_file_check.py"], + deps = [ + "//buildscripts/linter", + "//buildscripts/patch_builds", + dependency( + "structlog", + group = "evergreen", + ), + dependency( + "gitpython", + group = "evergreen", + ), + ], +) + # TODO(SERVER-105817): The following library is autogenerated, please split these out into individual python targets py_library( name = "all_python_files", diff --git a/buildscripts/large_file_check.py b/buildscripts/large_file_check.py index befe8e47ee1..d22c4b1814e 100755 --- a/buildscripts/large_file_check.py +++ b/buildscripts/large_file_check.py @@ -57,7 +57,11 @@ MONGO_REVISION_ENV_VAR = "REVISION" def _get_repos_and_revisions() -> Tuple[List[Repo], RevisionMap]: """Get the repo object and a map of revisions to compare against.""" - repos = [Repo(git.get_base_dir())] + repo_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", None) + if repo_dir: + repos = [Repo(repo_dir)] + else: + repos = [Repo(git.get_base_dir())] revision_map = generate_revision_map(repos, {"mongo": os.environ.get(MONGO_REVISION_ENV_VAR)}) return repos, revision_map diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml index a92c668cfc8..593cffc1a00 100644 --- a/etc/evergreen_yml_components/tasks/misc_tasks.yml +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -734,7 +734,7 @@ tasks: - func: "bazel run" vars: target: >- - lint //... --lint-yaml=${project} + lint //... --lint-yaml=${project} --large-files # Check that the mutational fuzzer can parse all JS filess. - name: lint_fuzzer_sanity_all @@ -767,33 +767,6 @@ tasks: - func: "setup jstestfuzz" - func: "lint fuzzer sanity patch" - - name: lint_large_files_check - tags: - [ - "assigned_to_jira_team_devprod_build", - "development_critical_single_variant", - ] - exec_timeout_secs: 600 # 10 minute timeout - commands: - - command: manifest.load - - func: "git get project and add git tag" - - func: "f_expansions_write" - - func: "kill processes" - - func: "cleanup environment" - - func: "set up venv" - - func: "upload pip requirements" - - func: "configure evergreen api credentials" - - command: subprocess.exec - type: test - params: - binary: bash - args: - - "./src/evergreen/run_python_script_with_report.sh" - - "lint-large-files-check" - - "buildscripts/large_file_check.py" - - "--exclude" - - "src/third_party/*" - - name: lint_pyright tags: [