mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-106979 Remove duplicate buildifier formatting (#37994) 

GitOrigin-RevId: 8eb8ad04d2acaa2316948916c4ae94f459d3d214
This commit is contained in:
Zack Winter 2025-07-02 15:06:05 -07:00 committed by MongoDB Bot
parent fde366b63a
commit 1583599e03
5 changed files with 1 additions and 338 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
buildscripts/BUILD.bazel
View File

@ -98,34 +98,6 @@ py_library(
],
)
py_library(
name = "download_buildifier",
srcs = [
"download_buildifier.py",
],
visibility = ["//visibility:public"],
deps = [
dependency(
"retry",
group = "testing",
),
],
)
py_binary(
name = "buildifier",
srcs = [
"buildifier.py",
],
main = "buildifier.py",
visibility = ["//visibility:public"],
deps = [
"download_buildifier",
"simple_report",
"unittest_grouper",
],
)
py_binary(
name = "legacy_commands_check",
srcs = [

173
buildscripts/buildifier.py
View File

@ -1,173 +0,0 @@
import argparse
import json
import os
import platform
import subprocess
import sys
sys.path.append(".")
from buildscripts import download_buildifier
from buildscripts.simple_report import make_report, put_report, try_combine_reports
from buildscripts.unittest_grouper import validate_bazel_groups
def find_all_failed(bin_path: str) -> list[str]:
# TODO(SERVER-81039): Remove once third_party libs can be compiled from the root directory.
ignored_paths = []
with open(os.path.join(os.curdir, ".bazelignore"), "r", encoding="utf-8") as file:
for line in file.readlines():
contents = line.split("#")[0].strip()
if contents:
ignored_paths.append(contents)
process = subprocess.run(
[bin_path, "--format=json", "--mode=check", "-r", "./"], check=True, capture_output=True
)
buildifier_results = json.loads(process.stdout)
if buildifier_results["success"]:
return []
return [
result["filename"]
for result in buildifier_results["files"]
if (
not result["formatted"]
and not any(
result["filename"].startswith(ignored_path) for ignored_path in ignored_paths
)
)
]
def lint_all(bin_path: str, generate_report: bool):
files = find_all_failed(bin_path)
result = lint(bin_path, files, generate_report)
validate_bazel_groups(generate_report=generate_report, fix=False)
return result
def fix_all(bin_path: str):
files = find_all_failed(bin_path)
fix(bin_path, files)
print("Checking unittest rules...")
validate_bazel_groups(generate_report=False, fix=True)
def lint(bin_path: str, files: list[str], generate_report: bool):
found_errors = False
for file in files:
process = subprocess.run(
[bin_path, "--format=json", "--mode=check", file], check=True, capture_output=True
)
result = json.loads(process.stdout)
if result["success"]:
continue
# This purposefully gives a exit code of 4 when there is a diff
process = subprocess.run(
[bin_path, "--mode=diff", file], capture_output=True, encoding="utf-8"
)
if process.returncode not in (0, 4):
raise RuntimeError()
diff = process.stdout
print(f"{file} has linting errors")
print(diff)
found_errors = True
if generate_report:
header = (
"There are linting errors in this file, fix them with one of the following commands:\n"
"python3 buildscripts/buildifier.py fix-all\n"
f"python3 buildscripts/buildifier.py fix {file}\n\n"
)
report = make_report(f"{file} warnings", json.dumps(result, indent=2), 1)
try_combine_reports(report)
put_report(report)
report = make_report(f"{file} diff", header + diff, 1)
try_combine_reports(report)
put_report(report)
print("Done linting files")
return not found_errors
def fix(bin_path: str, files: list[str]):
for file in files:
subprocess.run([bin_path, "--mode=fix", file], check=True)
print("Done fixing files")
def main():
default_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", ".")
os.chdir(default_dir)
parser = argparse.ArgumentParser(description="buildifier wrapper")
parser.add_argument(
"--binary-dir",
"-b",
type=str,
help="Path to the buildifier binary, defaults to looking in the current directory.",
default="",
)
parser.add_argument(
"--generate-report",
action="store_true",
help="Whether or not a report of the lint errors should be generated for evergreen.",
default=False,
)
parser.set_defaults(subcommand=None)
sub = parser.add_subparsers(title="buildifier subcommands", help="sub-command help")
lint_all_parser = sub.add_parser("lint-all", help="Lint all files")
lint_all_parser.set_defaults(subcommand="lint-all")
fix_all_parser = sub.add_parser("fix-all", help="Fix all files")
fix_all_parser.set_defaults(subcommand="fix-all")
lint_parser = sub.add_parser("lint", help="Lint specified list of files")
lint_parser.add_argument("files", nargs="+")
lint_parser.set_defaults(subcommand="lint")
lint_parser = sub.add_parser("fix", help="Fix specified list of files")
lint_parser.add_argument("files", nargs="+")
lint_parser.set_defaults(subcommand="fix")
args = parser.parse_args()
binary_name = "buildifier.exe" if platform.system() == "Windows" else "buildifier"
if args.binary_dir:
binary_path = os.path.join(args.binary_dir, binary_name)
else:
binary_path = os.path.join(os.curdir, binary_name)
if not os.path.exists(binary_path):
binary_dir = args.binary_dir if args.binary_dir else os.curdir
try:
download_buildifier.download(download_location=binary_dir)
except Exception as ex:
raise RuntimeError("Could not download and setup buildifier", ex)
subcommand = args.subcommand
if subcommand == "lint-all":
lint_all(binary_path, args.generate_report)
elif subcommand == "fix-all":
fix_all(binary_path)
elif subcommand == "lint":
lint(binary_path, args.files, args.generate_report)
elif subcommand == "fix":
fix(binary_path, args.files)
else:
# we purposefully do not use sub.choices.keys() so it does not print as a dict_keys object
choices = [key for key in sub.choices]
raise RuntimeError(f"One of the following subcommands must be specified: {choices}")
if os.environ.get("CI"):
local_usage = "python buildscripts/buildifier.py fix-all\npython buildscripts/buildifier.py fix-unittests\n"
sys.stderr.write("buildifier.py invocations with params for local usage:\n")
sys.stderr.write(local_usage)
with open("local-buildifier-invocation.txt", "w") as f:
f.write(local_usage)
if __name__ == "__main__":
main()

85
buildscripts/download_buildifier.py
View File

@ -1,85 +0,0 @@
import argparse
import os
import platform
import stat
import urllib.request
from retry import retry
BUILDIFIER_VERSION = "v6.4.0"
RELEASE_URL = (
f"https://mdb-build-public.s3.amazonaws.com/bazel-buildifier-binaries/{BUILDIFIER_VERSION}/"
)
@retry(tries=3, delay=5)
def _download_with_retry(*args, **kwargs):
return urllib.request.urlretrieve(*args, **kwargs)
def determine_platform():
syst = platform.system()
pltf = None
if syst == "Darwin":
pltf = "darwin"
elif syst == "Windows":
pltf = "windows"
elif syst == "Linux":
pltf = "linux"
else:
raise RuntimeError("Platform cannot be inferred.")
return pltf
def determine_architecture():
arch = None
machine = platform.machine()
if machine in ("AMD64", "x86_64"):
arch = "amd64"
elif machine in ("arm", "arm64", "aarch64"):
arch = "arm64"
else:
raise RuntimeError(f"Detected architecture is not supported: {machine}")
return arch
def download(download_location: str = "./"):
operating_system = determine_platform()
architechture = determine_architecture()
if operating_system == "windows" and architechture == "arm64":
raise RuntimeError("There are no published arm windows releases for buildifier.")
extension = ".exe" if operating_system == "windows" else ""
binary_name = f"buildifier-{operating_system}-{architechture}{extension}"
url = f"{RELEASE_URL}{binary_name}"
file_location = os.path.join(download_location, f"buildifier{extension}")
_download_with_retry(url, file_location)
print(f"Downloaded buildifier from {url} to {file_location}")
os.chmod(file_location, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
print(f"Set user executable permissions on {file_location}")
def main():
parser = argparse.ArgumentParser(
prog="DownloadBuildifier",
description="This downloads buildifier, it is intended for use in evergreen."
"This is our temperary solution to get bazel linting while we determine a "
"long-term solution for getting buildifier on evergreen/development hosts.",
)
parser.add_argument(
"--download-location",
"-f",
help="Name of directory to download the buildifier binary to.",
default="./",
)
args = parser.parse_known_args()
download(download_location=args[0].download_location)
if __name__ == "__main__":
main()

2
buildscripts/unittest_grouper.py
View File

@ -68,7 +68,7 @@ def download_buildozer(download_location: str = "./"):
operating_system = determine_platform()
architechture = determine_architecture()
if operating_system == "windows" and architechture == "arm64":
raise RuntimeError("There are no published arm windows releases for buildifier.")
raise RuntimeError("There are no published arm windows releases for buildozer.")
extension = ".exe" if operating_system == "windows" else ""
binary_name = f"buildozer-{operating_system}-{architechture}{extension}"

51
etc/evergreen_yml_components/tasks/misc_tasks.yml
View File

@ -884,57 +884,6 @@ tasks:
- "lint-poetry-lock"
- "evergreen/functions/poetry_lock_check.py"
- name: lint_bazel
tags:
[
"assigned_to_jira_team_devprod_build",
"development_critical_single_variant",
"lint",
]
commands:
- command: timeout.update
params:
# 20 mins
exec_timeout_secs: 1200
- func: "f_expansions_write"
- command: manifest.load
- func: "git get project and add git tag"
- func: "f_expansions_write"
- func: "kill processes"
- func: "cleanup environment"
- func: "set up venv"
- func: "upload pip requirements"
- func: "get engflow creds"
- command: subprocess.exec
params:
binary: bash
add_expansions_to_env: true
args:
- "src/evergreen/run_python_script.sh"
- "buildscripts/download_buildifier.py"
- command: subprocess.exec
params:
binary: bash
add_expansions_to_env: true
args:
- "src/evergreen/run_python_script_with_report.sh"
- "lint-bazel"
- "buildscripts/buildifier.py"
- "--generate-report"
- "--binary-dir=./"
- "lint-all"
- command: s3.put
params:
optional: true
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: src/local-buildifier-invocation.txt
remote_file: ${project}/${build_variant}/${revision}/local-buildifier-invocation-${task_id}-${execution}.txt
bucket: mciuploads
permissions: public-read
content_type: text/plain
display_name: Buildifier.py Invocation for Local Usage
# TODO: DEVPROD-9047 Enable after the issue with getting third party file is resolved
- name: lint_sbom
tags: