mirror of https://github.com/mongodb/mongo
SERVER-109370 Use S3 Binary interface everywhere that the codebase downloads binaries from s3 (#43051)
GitOrigin-RevId: 328069792b90087bc410d4182f4e2a8b542edd7e
This commit is contained in:
parent
af84ecdcca
commit
9887b87fb4
|
|
@ -1,618 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Clang format script that provides the following.
|
||||
|
||||
1. Ability to grab binaries where possible from LLVM.
|
||||
2. Ability to download binaries from MongoDB cache for clang-format.
|
||||
3. Validates clang-format is the right version.
|
||||
4. Has support for checking which files are to be checked.
|
||||
5. Supports validating and updating a set of files to the right coding style.
|
||||
"""
|
||||
|
||||
import difflib
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from optparse import OptionParser
|
||||
|
||||
import structlog
|
||||
|
||||
# Get relative imports to work when the package is not installed on the PYTHONPATH.
|
||||
if __name__ == "__main__" and __package__ is None:
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
|
||||
|
||||
from buildscripts.linter import git, parallel
|
||||
from buildscripts.linter.filediff import gather_changed_files_for_lint
|
||||
from buildscripts.mongo_toolchain import MongoToolchainError, get_mongo_toolchain
|
||||
|
||||
##############################################################################
|
||||
#
|
||||
# Constants for clang-format
|
||||
#
|
||||
#
|
||||
|
||||
# Expected version of clang-format
|
||||
CLANG_FORMAT_VERSION = "19.1.7"
|
||||
CLANG_FORMAT_SHORT_VERSION = "19.1"
|
||||
CLANG_FORMAT_SHORTER_VERSION = "191"
|
||||
|
||||
# Name of clang-format as a binary
|
||||
CLANG_FORMAT_PROGNAME = "clang-format"
|
||||
|
||||
TOOLCHAIN_VERSION = "v5"
|
||||
|
||||
CLANG_FORMAT_HTTP_DARWIN_CACHE = (
|
||||
"https://mdb-build-public.s3.amazonaws.com/clang-format-osx-binaries/clang-format-19.1.7"
|
||||
)
|
||||
|
||||
try:
|
||||
toolchain = get_mongo_toolchain(version=TOOLCHAIN_VERSION)
|
||||
CLANG_FORMAT_TOOLCHAIN_PATH = toolchain.get_tool_path(CLANG_FORMAT_PROGNAME)
|
||||
except MongoToolchainError:
|
||||
# This isn't fatal, we'll try to get clang-format using other methods if we don't find the
|
||||
# mongo toolchain.
|
||||
CLANG_FORMAT_TOOLCHAIN_PATH = ""
|
||||
|
||||
|
||||
##############################################################################
|
||||
def callo(args, **kwargs):
|
||||
"""Call a program, and capture its output."""
|
||||
return subprocess.check_output(args, **kwargs).decode("utf-8")
|
||||
|
||||
|
||||
def get_clang_format_from_cache(url, dest_file):
|
||||
"""Get clang-format from mongodb's cache."""
|
||||
# Download from file
|
||||
print(
|
||||
"Downloading clang-format %s from %s, saving to %s" % (CLANG_FORMAT_VERSION, url, dest_file)
|
||||
)
|
||||
|
||||
# Retry download up to 5 times.
|
||||
num_tries = 5
|
||||
for attempt in range(num_tries):
|
||||
try:
|
||||
resp = urllib.request.urlopen(url)
|
||||
with open(dest_file, "wb") as fh:
|
||||
fh.write(resp.read())
|
||||
break
|
||||
except urllib.error.URLError:
|
||||
if attempt == num_tries - 1:
|
||||
raise
|
||||
continue
|
||||
|
||||
|
||||
class ClangFormat(object):
|
||||
"""ClangFormat class."""
|
||||
|
||||
def __init__(self, path, cache_dir):
|
||||
"""Initialize ClangFormat."""
|
||||
self.path = None
|
||||
|
||||
# Check the clang-format the user specified
|
||||
if path is not None:
|
||||
self.path = path
|
||||
if not self._validate_version():
|
||||
print(
|
||||
"WARNING: Could not find clang-format in the user specified path %s"
|
||||
% (self.path)
|
||||
)
|
||||
self.path = None
|
||||
|
||||
# Check the environment variable
|
||||
if self.path is None:
|
||||
if "MONGO_CLANG_FORMAT" in os.environ:
|
||||
self.path = os.environ["MONGO_CLANG_FORMAT"]
|
||||
if not self._validate_version():
|
||||
self.path = None
|
||||
|
||||
# Check for the binary in the expected toolchain directory on non-windows systems
|
||||
if self.path is None:
|
||||
if sys.platform != "win32":
|
||||
if os.path.exists(CLANG_FORMAT_TOOLCHAIN_PATH):
|
||||
self.path = CLANG_FORMAT_TOOLCHAIN_PATH
|
||||
if not self._validate_version():
|
||||
self.path = None
|
||||
|
||||
# Check the users' PATH environment variable now
|
||||
if self.path is None:
|
||||
# Check for various versions staring with binaries with version specific suffixes in the
|
||||
# user's path
|
||||
programs = list(
|
||||
map(
|
||||
lambda program: program + ".exe" if sys.platform == "win32" else program,
|
||||
[
|
||||
CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_VERSION,
|
||||
CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_SHORT_VERSION,
|
||||
CLANG_FORMAT_PROGNAME + CLANG_FORMAT_SHORTER_VERSION,
|
||||
CLANG_FORMAT_PROGNAME,
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
directories_to_check = os.environ["PATH"].split(os.pathsep)
|
||||
|
||||
# If Windows, try to grab it from Program Files
|
||||
# Check both native Program Files and WOW64 version
|
||||
if sys.platform == "win32":
|
||||
programfiles = [
|
||||
os.environ["ProgramFiles"],
|
||||
os.environ["ProgramFiles(x86)"],
|
||||
]
|
||||
|
||||
directories_to_check += [os.path.join(p, "LLVM\\bin\\") for p in programfiles]
|
||||
|
||||
for ospath in directories_to_check:
|
||||
for program in programs:
|
||||
self.path = os.path.join(ospath, program)
|
||||
if os.path.exists(self.path) and self._validate_version():
|
||||
break
|
||||
self.path = None
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
break
|
||||
|
||||
# Have not found it yet, download it from the web
|
||||
if self.path is None:
|
||||
if not os.path.isdir(cache_dir):
|
||||
os.makedirs(cache_dir)
|
||||
|
||||
clang_format_progname_ext = ".exe" if sys.platform == "win32" else ""
|
||||
self.path = os.path.join(
|
||||
cache_dir,
|
||||
CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_VERSION + clang_format_progname_ext,
|
||||
)
|
||||
|
||||
# Download a new version if the cache is empty or stale and set permissions (0755)
|
||||
if not os.path.isfile(self.path) or not self._validate_version():
|
||||
if sys.platform == "darwin":
|
||||
get_clang_format_from_cache(CLANG_FORMAT_HTTP_DARWIN_CACHE, self.path)
|
||||
os.chmod(
|
||||
self.path,
|
||||
stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH,
|
||||
)
|
||||
else:
|
||||
print(
|
||||
"ERROR: clang_format.py does not support downloading clang-format "
|
||||
+ "on this platform, please install clang-format "
|
||||
+ CLANG_FORMAT_VERSION
|
||||
)
|
||||
|
||||
# Validate we have the correct version
|
||||
# We only can fail here if the user specified a clang-format binary and it is the wrong
|
||||
# version
|
||||
if not self._validate_version():
|
||||
print("ERROR: exiting because of previous warning.")
|
||||
sys.exit(1)
|
||||
|
||||
self.print_lock = threading.Lock()
|
||||
|
||||
def _validate_version(self):
|
||||
"""Validate clang-format is the expected version."""
|
||||
cf_version = callo([self.path, "--version"])
|
||||
|
||||
if CLANG_FORMAT_VERSION in cf_version:
|
||||
return True
|
||||
|
||||
print(
|
||||
"WARNING: clang-format with incorrect version found at "
|
||||
+ self.path
|
||||
+ " version: "
|
||||
+ cf_version
|
||||
)
|
||||
|
||||
return False
|
||||
|
||||
def _lint(self, file_name, print_diff):
|
||||
"""Check the specified file has the correct format."""
|
||||
with open(file_name, "rb") as original_text:
|
||||
original_file = original_text.read().decode("utf-8")
|
||||
|
||||
original_text.seek(0)
|
||||
|
||||
# Get formatted file as clang-format would format the file
|
||||
formatted_file = callo(
|
||||
[
|
||||
self.path,
|
||||
"--assume-filename="
|
||||
+ (file_name if not file_name.endswith(".h") else file_name + "pp"),
|
||||
"--style=file",
|
||||
],
|
||||
stdin=original_text,
|
||||
)
|
||||
|
||||
if original_file != formatted_file:
|
||||
if print_diff:
|
||||
original_lines = original_file.splitlines()
|
||||
formatted_lines = formatted_file.splitlines()
|
||||
result = difflib.unified_diff(original_lines, formatted_lines)
|
||||
|
||||
# Take a lock to ensure diffs do not get mixed when printed to the screen
|
||||
with self.print_lock:
|
||||
print("ERROR: Found diff for " + file_name)
|
||||
print("To fix formatting errors, run `buildscripts/clang_format.py format`")
|
||||
for line in result:
|
||||
print(line.rstrip())
|
||||
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def lint(self, file_name):
|
||||
"""Check the specified file has the correct format."""
|
||||
return self._lint(file_name, print_diff=True)
|
||||
|
||||
def format(self, file_name):
|
||||
"""Update the format of the specified file."""
|
||||
if self._lint(file_name, print_diff=False):
|
||||
return True
|
||||
|
||||
# Update the file with clang-format
|
||||
# We have to tell `clang-format` to format on standard input due to its file type
|
||||
# determiner. `--assume-filename` doesn't work directly on files, but only on standard
|
||||
# input. Thus we have to open the file as the subprocess's standard input. Then we record
|
||||
# that formatted standard output back into the file. We can't use the `-i` option, due to
|
||||
# the fact that `clang-format` believes that many of our C++ headers are Objective-C code.
|
||||
formatted = True
|
||||
with open(file_name, "rb") as source_stream:
|
||||
try:
|
||||
reformatted_text = subprocess.check_output(
|
||||
[
|
||||
self.path,
|
||||
"--assume-filename="
|
||||
+ (file_name if not file_name.endswith(".h") else file_name + "pp"),
|
||||
"--style=file",
|
||||
],
|
||||
stdin=source_stream,
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
formatted = False
|
||||
|
||||
if formatted:
|
||||
with open(file_name, "wb") as output_stream:
|
||||
output_stream.write(reformatted_text)
|
||||
|
||||
# Version 3.8 generates files like foo.cpp~RF83372177.TMP when it formats foo.cpp
|
||||
# on Windows, we must clean these up
|
||||
if sys.platform == "win32":
|
||||
glob_pattern = file_name + "*.TMP"
|
||||
for fglob in glob.glob(glob_pattern):
|
||||
os.unlink(fglob)
|
||||
|
||||
return formatted
|
||||
|
||||
|
||||
FILES_RE = re.compile("\\.(h|hpp|ipp|cpp)$")
|
||||
TPL_FILES_RE = re.compile("\\.tpl\\.")
|
||||
|
||||
|
||||
def is_interesting_file(file_name):
|
||||
"""Return true if this file should be checked."""
|
||||
return (
|
||||
(
|
||||
file_name.startswith("src")
|
||||
and not file_name.startswith("src/third_party/")
|
||||
and not file_name.startswith("src/mongo/gotools/")
|
||||
and not file_name.startswith("src/mongo/db/modules/enterprise/src/streams/third_party")
|
||||
and not file_name.startswith("src/streams/third_party")
|
||||
)
|
||||
and FILES_RE.search(file_name)
|
||||
and not TPL_FILES_RE.search(file_name)
|
||||
)
|
||||
|
||||
|
||||
def get_list_from_lines(lines):
|
||||
"""Convert a string containing a series of lines into a list of strings."""
|
||||
return [line.rstrip() for line in lines.splitlines()]
|
||||
|
||||
|
||||
def _get_build_dir():
|
||||
"""Return the location of the default clang cache directory."""
|
||||
return os.path.join(git.get_base_dir(), ".clang_format_cache")
|
||||
|
||||
|
||||
def _lint_files(clang_format, files):
|
||||
"""Lint a list of files with clang-format."""
|
||||
clang_format = ClangFormat(clang_format, _get_build_dir())
|
||||
|
||||
lint_clean = parallel.parallel_process([os.path.abspath(f) for f in files], clang_format.lint)
|
||||
|
||||
if not lint_clean:
|
||||
print("ERROR: Source code does not match required source formatting style")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def lint_patch(clang_format, infile):
|
||||
"""Lint patch command entry point."""
|
||||
files = git.get_files_to_check_from_patch(infile, is_interesting_file)
|
||||
|
||||
# Patch may have files that we do not want to check which is fine
|
||||
if files:
|
||||
_lint_files(clang_format, files)
|
||||
|
||||
|
||||
def lint_git_diff(clang_format):
|
||||
"""
|
||||
Lint the files that have changes since the last git commit.
|
||||
|
||||
:param clang_format: Path to clang_format command.
|
||||
"""
|
||||
files = gather_changed_files_for_lint(is_interesting_file)
|
||||
|
||||
if files:
|
||||
_lint_files(clang_format, files)
|
||||
|
||||
|
||||
def lint(clang_format):
|
||||
"""Lint files command entry point."""
|
||||
files = git.get_files_to_check([], is_interesting_file)
|
||||
|
||||
_lint_files(clang_format, files)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def lint_all(clang_format):
|
||||
"""Lint files command entry point based on working tree."""
|
||||
files = git.get_files_to_check_working_tree(is_interesting_file)
|
||||
|
||||
_lint_files(clang_format, files)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _format_files(clang_format, files):
|
||||
"""Format a list of files with clang-format."""
|
||||
clang_format = ClangFormat(clang_format, _get_build_dir())
|
||||
|
||||
format_clean = parallel.parallel_process(
|
||||
[os.path.abspath(f) for f in files], clang_format.format
|
||||
)
|
||||
|
||||
if not format_clean:
|
||||
print("ERROR: failed to format files")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def format_func(clang_format):
|
||||
"""Format files command entry point."""
|
||||
files = git.get_files_to_check([], is_interesting_file)
|
||||
|
||||
_format_files(clang_format, files)
|
||||
|
||||
|
||||
def format_one(clang_format, filename):
|
||||
"""Format file command entry point."""
|
||||
_format_files(clang_format, [filename])
|
||||
|
||||
|
||||
def format_my_func(clang_format, origin_branch):
|
||||
"""My Format files command entry point."""
|
||||
files = git.get_my_files_to_check(is_interesting_file, origin_branch)
|
||||
files = [f for f in files if os.path.exists(f)]
|
||||
|
||||
_format_files(clang_format, files)
|
||||
|
||||
|
||||
def reformat_branch(clang_format, commit_prior_to_reformat, commit_after_reformat):
|
||||
"""Reformat a branch made before a clang-format run."""
|
||||
clang_format = ClangFormat(clang_format, _get_build_dir())
|
||||
|
||||
if os.getcwd() != git.get_base_dir():
|
||||
raise ValueError("reformat-branch must be run from the repo root")
|
||||
|
||||
if not os.path.exists("buildscripts/clang_format.py"):
|
||||
raise ValueError("reformat-branch is only supported in the mongo repo")
|
||||
|
||||
repo = git.Repo(git.get_base_dir())
|
||||
|
||||
# Validate that user passes valid commits
|
||||
if not repo.is_commit(commit_prior_to_reformat):
|
||||
raise ValueError(
|
||||
"Commit Prior to Reformat '%s' is not a valid commit in this repo"
|
||||
% commit_prior_to_reformat
|
||||
)
|
||||
|
||||
if not repo.is_commit(commit_after_reformat):
|
||||
raise ValueError(
|
||||
"Commit After Reformat '%s' is not a valid commit in this repo" % commit_after_reformat
|
||||
)
|
||||
|
||||
if not repo.is_ancestor(commit_prior_to_reformat, commit_after_reformat):
|
||||
raise ValueError(
|
||||
(
|
||||
"Commit Prior to Reformat '%s' is not a valid ancestor of Commit After"
|
||||
+ " Reformat '%s' in this repo"
|
||||
)
|
||||
% (commit_prior_to_reformat, commit_after_reformat)
|
||||
)
|
||||
|
||||
# Validate the user is on a local branch that has the right merge base
|
||||
if repo.is_detached():
|
||||
raise ValueError("You must not run this script in a detached HEAD state")
|
||||
|
||||
# Validate the user has no pending changes
|
||||
if repo.is_working_tree_dirty():
|
||||
raise ValueError(
|
||||
"Your working tree has pending changes. You must have a clean working tree before proceeding."
|
||||
)
|
||||
|
||||
merge_base = repo.get_merge_base(["HEAD", commit_prior_to_reformat])
|
||||
|
||||
if not merge_base == commit_prior_to_reformat:
|
||||
raise ValueError(
|
||||
"Please rebase to '%s' and resolve all conflicts before running this script"
|
||||
% (commit_prior_to_reformat)
|
||||
)
|
||||
|
||||
# We assume the target branch is master, it could be a different branch if needed for testing
|
||||
merge_base = repo.get_merge_base(["HEAD", "master"])
|
||||
|
||||
if not merge_base == commit_prior_to_reformat:
|
||||
raise ValueError(
|
||||
"This branch appears to already have advanced too far through the merge process"
|
||||
)
|
||||
|
||||
# Everything looks good so lets start going through all the commits
|
||||
branch_name = repo.get_branch_name()
|
||||
new_branch = "%s-reformatted" % branch_name
|
||||
|
||||
if repo.does_branch_exist(new_branch):
|
||||
raise ValueError(
|
||||
"The branch '%s' already exists. Please delete the branch '%s', or rename the current branch."
|
||||
% (new_branch, new_branch)
|
||||
)
|
||||
|
||||
commits = get_list_from_lines(
|
||||
repo.git_log(
|
||||
[
|
||||
"--reverse",
|
||||
"--no-show-signature",
|
||||
"--pretty=format:%H",
|
||||
"%s..HEAD" % commit_prior_to_reformat,
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
previous_commit_base = commit_after_reformat
|
||||
|
||||
# Go through all the commits the user made on the local branch and migrate to a new branch
|
||||
# that is based on post_reformat commits instead
|
||||
for commit_hash in commits:
|
||||
repo.git_checkout(["--quiet", commit_hash])
|
||||
|
||||
deleted_files = []
|
||||
|
||||
# Format each of the files by checking out just a single commit from the user's branch
|
||||
commit_files = get_list_from_lines(repo.git_diff(["HEAD~", "--name-only"]))
|
||||
|
||||
for commit_file in commit_files:
|
||||
# Format each file needed if it was not deleted
|
||||
if not os.path.exists(commit_file):
|
||||
print(
|
||||
"Skipping file '%s' since it has been deleted in commit '%s'"
|
||||
% (commit_file, commit_hash)
|
||||
)
|
||||
deleted_files.append(commit_file)
|
||||
continue
|
||||
|
||||
if is_interesting_file(commit_file):
|
||||
clang_format.format(commit_file)
|
||||
else:
|
||||
print(
|
||||
"Skipping file '%s' since it is not a file clang_format should format"
|
||||
% commit_file
|
||||
)
|
||||
|
||||
# Check if anything needed reformatting, and if so amend the commit
|
||||
if not repo.is_working_tree_dirty():
|
||||
print("Commit %s needed no reformatting" % commit_hash)
|
||||
else:
|
||||
repo.git_commit(["--all", "--amend", "--no-edit"])
|
||||
|
||||
# Rebase our new commit on top the post-reformat commit
|
||||
previous_commit = repo.git_rev_parse(["HEAD"])
|
||||
|
||||
# Checkout the new branch with the reformatted commits
|
||||
# Note: we will not name as a branch until we are done with all commits on the local branch
|
||||
repo.git_checkout(["--quiet", previous_commit_base])
|
||||
|
||||
# Copy each file from the reformatted commit on top of the post reformat
|
||||
diff_files = get_list_from_lines(
|
||||
repo.git_diff(["%s~..%s" % (previous_commit, previous_commit), "--name-only"])
|
||||
)
|
||||
|
||||
for diff_file in diff_files:
|
||||
# If the file was deleted in the commit we are reformatting, we need to delete it again
|
||||
if diff_file in deleted_files:
|
||||
repo.git_rm(["--ignore-unmatch", diff_file])
|
||||
continue
|
||||
|
||||
# The file has been added or modified, continue as normal
|
||||
file_contents = repo.git_show(["%s:%s" % (previous_commit, diff_file)])
|
||||
|
||||
root_dir = os.path.dirname(diff_file)
|
||||
if root_dir and not os.path.exists(root_dir):
|
||||
os.makedirs(root_dir)
|
||||
|
||||
with open(diff_file, "w+", encoding="utf-8") as new_file:
|
||||
new_file.write(file_contents)
|
||||
|
||||
repo.git_add([diff_file])
|
||||
|
||||
# Create a new commit onto clang-formatted branch
|
||||
repo.git_commit(["--reuse-message=%s" % previous_commit, "--no-gpg-sign", "--allow-empty"])
|
||||
|
||||
previous_commit_base = repo.git_rev_parse(["HEAD"])
|
||||
|
||||
# Create a new branch to mark the hashes we have been using
|
||||
repo.git_checkout(["-b", new_branch])
|
||||
|
||||
print("reformat-branch is done running.\n")
|
||||
print(
|
||||
"A copy of your branch has been made named '%s', and formatted with clang-format.\n"
|
||||
% new_branch
|
||||
)
|
||||
print("The original branch has been left unchanged.")
|
||||
print("The next step is to rebase the new branch on 'master'.")
|
||||
|
||||
|
||||
def usage():
|
||||
"""Print usage."""
|
||||
print(
|
||||
"clang_format.py supports 6 commands (lint, lint-all, lint-patch, format, format-my, reformat-branch)."
|
||||
)
|
||||
print("\nformat-my <origin branch>")
|
||||
print(" <origin branch> - upstream branch to compare against")
|
||||
|
||||
|
||||
def main():
|
||||
"""Execute Main entry point."""
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
|
||||
structlog.configure(logger_factory=structlog.stdlib.LoggerFactory())
|
||||
|
||||
parser = OptionParser()
|
||||
parser.add_option("-c", "--clang-format", type="string", dest="clang_format")
|
||||
|
||||
(options, args) = parser.parse_args(args=sys.argv)
|
||||
|
||||
if len(args) > 1:
|
||||
command = args[1]
|
||||
|
||||
if command == "lint":
|
||||
lint(options.clang_format)
|
||||
elif command == "lint-all":
|
||||
lint_all(options.clang_format)
|
||||
elif command == "lint-patch":
|
||||
lint_patch(options.clang_format, args[2:])
|
||||
elif command == "lint-git-diff":
|
||||
lint_git_diff(options.clang_format)
|
||||
elif command == "format":
|
||||
format_func(options.clang_format)
|
||||
elif command == "format-one":
|
||||
format_one(options.clang_format, args[2])
|
||||
elif command == "format-my":
|
||||
format_my_func(options.clang_format, args[2] if len(args) > 2 else "origin/master")
|
||||
elif command == "reformat-branch":
|
||||
if len(args) < 3:
|
||||
print(
|
||||
"ERROR: reformat-branch takes two parameters: commit_prior_to_reformat commit_after_reformat"
|
||||
)
|
||||
return
|
||||
|
||||
reformat_branch(options.clang_format, args[2], args[3])
|
||||
else:
|
||||
usage()
|
||||
else:
|
||||
usage()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -23,4 +23,11 @@ S3_SHA256_HASHES = {
|
|||
"https://mdb-build-public.s3.amazonaws.com/fd-binaries/v10.3.0/fd-linux-amd64": "9f48273b6c780a5f4f084ef30bc67d98cbd7d10c55c4605cf3a6ee29b741af87",
|
||||
"https://mdb-build-public.s3.amazonaws.com/fd-binaries/v10.3.0/fd-linux-arm64": "d0fc407937b8a8aec44f3a80b4a08219ba61dde234590358abb168b44478d493",
|
||||
"https://mdb-build-public.s3.amazonaws.com/fd-binaries/v10.3.0/fd-windows-amd64.exe": "fd3d4853da7a319a604e1cb03ede88cbf584edd12b89a0991871fb4d9cd3ba5b",
|
||||
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_darwin_x64.gz": "30eb3ed0ef541be6eea051a9097049f4b91bb565f5a0faf59ba317c061e3c3e6",
|
||||
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_linux_arm64.gz": "b047e429f6f455febc970c9f59c75a92986cb638456238d71716fe2810b9177f",
|
||||
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_linux_s390x.gz": "ae3c989c0a867bec6ba27d056019039be544d80cf154f4bdaa065bb9e825675c",
|
||||
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_linux_x64.gz": "184065d8e70cfa3f98eae399b2444688775133733e4c4daa7a32ffdaa8602faa",
|
||||
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_rhel8_ppc64le.gz": "ad0bc6188223abc115a653945ada288ac4f6232ec7226b975bb1efda8b898095",
|
||||
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_rhel9_ppc64le.gz": "73805acc4786df9b414b1b28ce40680cd6c9a9a0fb58712928711ee6786c939d",
|
||||
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_windows_x64.gz": "f138b09612d143fc67c8cc8efb8cfbae13ce50cf8049a7d802c5cd262176346a",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -372,16 +372,6 @@ def run_smoke_tests(
|
|||
)
|
||||
|
||||
formatters = [
|
||||
runner.command(
|
||||
name="clang format",
|
||||
args=[
|
||||
MONGO_PYTHON_INTERPRETER,
|
||||
ROOT.joinpath("buildscripts", "clang_format.py"),
|
||||
"format-my",
|
||||
upstream_branch,
|
||||
],
|
||||
log_file="clang_format.log",
|
||||
),
|
||||
runner.command(
|
||||
name="misc. lint",
|
||||
args=[
|
||||
|
|
|
|||
|
|
@ -1003,17 +1003,11 @@ python buildscripts/eslint.py fix # auto-fix js code
|
|||
|
||||
All code changes must be formatted by
|
||||
[clang-format](http://clang.llvm.org/docs/ClangFormat.html) before they are
|
||||
checked in. Use `buildscripts/clang_format.py` to reformat C++ and JS code.
|
||||
checked in. Use `bazel run format` to reformat C++ and JS code.
|
||||
Clang-format is a C/C++ & JS code formatting tool that uses the config files
|
||||
located at `src/mongo/.clang-format` and `jstests/.clang-format` to control the
|
||||
format of the code. The version and configuration of clang-format is selected by
|
||||
`buildscripts/clang_format.py`.
|
||||
|
||||
```sh
|
||||
python buildscripts/clang_format.py lint # check
|
||||
python buildscripts/clang_format.py format # format all code
|
||||
python buildscripts/clang_format.py format-my # reformat only changed files
|
||||
```
|
||||
`bazel run format`.
|
||||
|
||||
Plugins are available for most editors that will automatically run clang-format
|
||||
on file save.
|
||||
|
|
|
|||
|
|
@ -2,17 +2,6 @@
|
|||
|
||||
## C++ Linters
|
||||
|
||||
### `clang-format`
|
||||
|
||||
The `buildscripts/clang_format.py` wrapper script runs the `clang-format` linter. You can see the
|
||||
usage message for the wrapper by running `buildscripts/clang_format.py --help`.
|
||||
|
||||
Ex: `buildscripts/clang_format.py lint`
|
||||
|
||||
| Linter | Configuration File(s) | Help Command | Documentation |
|
||||
| -------------- | --------------------- | --------------------- | -------------------------------------------------------------------------------------------- |
|
||||
| `clang-format` | `.clang-format` | `clang-format --help` | [https://clang.llvm.org/docs/ClangFormat.html](https://clang.llvm.org/docs/ClangFormat.html) |
|
||||
|
||||
### `clang-tidy`
|
||||
|
||||
The `buildscripts/clang_tidy.py` shell script runs the `clang-tidy` linter. In order to run
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
load("@poetry//:dependencies.bzl", "dependency")
|
||||
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
py_binary(
|
||||
|
|
@ -563,3 +565,18 @@ py_library(
|
|||
srcs = glob(["*.py"]),
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
py_binary(
|
||||
name = "download_db_contrib_tool",
|
||||
srcs = ["download_db_contrib_tool.py"],
|
||||
main = "download_db_contrib_tool.py",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
dependency(
|
||||
"pyyaml",
|
||||
group = "core",
|
||||
),
|
||||
"//buildscripts/s3_binary:download",
|
||||
"//buildscripts/util",
|
||||
],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,12 +5,11 @@ import platform
|
|||
import re
|
||||
import stat
|
||||
import sys
|
||||
from urllib.request import urlretrieve
|
||||
|
||||
import retry
|
||||
|
||||
mongo_path = pathlib.Path(__file__).parents[1]
|
||||
sys.path.append(str(mongo_path))
|
||||
|
||||
from buildscripts.s3_binary.download import download_s3_binary
|
||||
from buildscripts.util.expansions import get_expansion
|
||||
|
||||
DB_CONTRIB_TOOL_VERSION = "v2.0.1"
|
||||
|
|
@ -50,16 +49,18 @@ def get_binary_name() -> str:
|
|||
return binary_name
|
||||
|
||||
|
||||
@retry.retry(tries=3, delay=3)
|
||||
def download_binary(url: str, path: str) -> None:
|
||||
urlretrieve(url, path)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", "."))
|
||||
|
||||
binary_name = get_binary_name()
|
||||
gz_name = f"{binary_name}.gz"
|
||||
binary_url = f"{RELEASE_URL}{gz_name}"
|
||||
download_binary(binary_url, gz_name)
|
||||
if not download_s3_binary(
|
||||
s3_path=binary_url,
|
||||
local_path=gz_name,
|
||||
):
|
||||
print("Failed to download db-contrib-tool binary")
|
||||
return 1
|
||||
# extract the binary
|
||||
with gzip.open(gz_name, "rb") as fin:
|
||||
with open(binary_name, "wb") as fout:
|
||||
|
|
|
|||
|
|
@ -280,7 +280,7 @@ def main(
|
|||
print(f"Modified {len(idl_files_modified)} IDL files to add `mod_visibility: private`")
|
||||
for file_path in idl_files_modified:
|
||||
print(f" {file_path}")
|
||||
print("Please run 'buildscripts/clang_format.py format-my' to clean up changes to headers")
|
||||
print("Please run 'bazel run format' to clean up changes to headers")
|
||||
else:
|
||||
print(
|
||||
f"\nDry run complete. Would modify {len(headers_modified) + len(idl_files_modified)} files."
|
||||
|
|
|
|||
Loading…
Reference in New Issue