mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-109370 Use S3 Binary interface everywhere that the codebase downloads binaries from s3 (#43051) 

GitOrigin-RevId: 328069792b90087bc410d4182f4e2a8b542edd7e
This commit is contained in:
Zack Winter 2025-10-27 19:26:08 -07:00 committed by MongoDB Bot
parent af84ecdcca
commit 9887b87fb4
8 changed files with 37 additions and 657 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

618
buildscripts/clang_format.py
View File

@ -1,618 +0,0 @@
#!/usr/bin/env python3
"""Clang format script that provides the following.
1. Ability to grab binaries where possible from LLVM.
2. Ability to download binaries from MongoDB cache for clang-format.
3. Validates clang-format is the right version.
4. Has support for checking which files are to be checked.
5. Supports validating and updating a set of files to the right coding style.
"""
import difflib
import glob
import logging
import os
import re
import stat
import subprocess
import sys
import threading
import urllib.error
import urllib.parse
import urllib.request
from optparse import OptionParser
import structlog
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
from buildscripts.linter import git, parallel
from buildscripts.linter.filediff import gather_changed_files_for_lint
from buildscripts.mongo_toolchain import MongoToolchainError, get_mongo_toolchain
##############################################################################
#
# Constants for clang-format
#
#
# Expected version of clang-format
CLANG_FORMAT_VERSION = "19.1.7"
CLANG_FORMAT_SHORT_VERSION = "19.1"
CLANG_FORMAT_SHORTER_VERSION = "191"
# Name of clang-format as a binary
CLANG_FORMAT_PROGNAME = "clang-format"
TOOLCHAIN_VERSION = "v5"
CLANG_FORMAT_HTTP_DARWIN_CACHE = (
"https://mdb-build-public.s3.amazonaws.com/clang-format-osx-binaries/clang-format-19.1.7"
)
try:
toolchain = get_mongo_toolchain(version=TOOLCHAIN_VERSION)
CLANG_FORMAT_TOOLCHAIN_PATH = toolchain.get_tool_path(CLANG_FORMAT_PROGNAME)
except MongoToolchainError:
# This isn't fatal, we'll try to get clang-format using other methods if we don't find the
# mongo toolchain.
CLANG_FORMAT_TOOLCHAIN_PATH = ""
##############################################################################
def callo(args, **kwargs):
"""Call a program, and capture its output."""
return subprocess.check_output(args, **kwargs).decode("utf-8")
def get_clang_format_from_cache(url, dest_file):
"""Get clang-format from mongodb's cache."""
# Download from file
print(
"Downloading clang-format %s from %s, saving to %s" % (CLANG_FORMAT_VERSION, url, dest_file)
)
# Retry download up to 5 times.
num_tries = 5
for attempt in range(num_tries):
try:
resp = urllib.request.urlopen(url)
with open(dest_file, "wb") as fh:
fh.write(resp.read())
break
except urllib.error.URLError:
if attempt == num_tries - 1:
raise
continue
class ClangFormat(object):
"""ClangFormat class."""
def __init__(self, path, cache_dir):
"""Initialize ClangFormat."""
self.path = None
# Check the clang-format the user specified
if path is not None:
self.path = path
if not self._validate_version():
print(
"WARNING: Could not find clang-format in the user specified path %s"
% (self.path)
)
self.path = None
# Check the environment variable
if self.path is None:
if "MONGO_CLANG_FORMAT" in os.environ:
self.path = os.environ["MONGO_CLANG_FORMAT"]
if not self._validate_version():
self.path = None
# Check for the binary in the expected toolchain directory on non-windows systems
if self.path is None:
if sys.platform != "win32":
if os.path.exists(CLANG_FORMAT_TOOLCHAIN_PATH):
self.path = CLANG_FORMAT_TOOLCHAIN_PATH
if not self._validate_version():
self.path = None
# Check the users' PATH environment variable now
if self.path is None:
# Check for various versions staring with binaries with version specific suffixes in the
# user's path
programs = list(
map(
lambda program: program + ".exe" if sys.platform == "win32" else program,
[
CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_VERSION,
CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_SHORT_VERSION,
CLANG_FORMAT_PROGNAME + CLANG_FORMAT_SHORTER_VERSION,
CLANG_FORMAT_PROGNAME,
],
)
)
directories_to_check = os.environ["PATH"].split(os.pathsep)
# If Windows, try to grab it from Program Files
# Check both native Program Files and WOW64 version
if sys.platform == "win32":
programfiles = [
os.environ["ProgramFiles"],
os.environ["ProgramFiles(x86)"],
]
directories_to_check += [os.path.join(p, "LLVM\\bin\\") for p in programfiles]
for ospath in directories_to_check:
for program in programs:
self.path = os.path.join(ospath, program)
if os.path.exists(self.path) and self._validate_version():
break
self.path = None
continue
else:
continue
break
# Have not found it yet, download it from the web
if self.path is None:
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
clang_format_progname_ext = ".exe" if sys.platform == "win32" else ""
self.path = os.path.join(
cache_dir,
CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_VERSION + clang_format_progname_ext,
)
# Download a new version if the cache is empty or stale and set permissions (0755)
if not os.path.isfile(self.path) or not self._validate_version():
if sys.platform == "darwin":
get_clang_format_from_cache(CLANG_FORMAT_HTTP_DARWIN_CACHE, self.path)
os.chmod(
self.path,
stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH,
)
else:
print(
"ERROR: clang_format.py does not support downloading clang-format "
+ "on this platform, please install clang-format "
+ CLANG_FORMAT_VERSION
)
# Validate we have the correct version
# We only can fail here if the user specified a clang-format binary and it is the wrong
# version
if not self._validate_version():
print("ERROR: exiting because of previous warning.")
sys.exit(1)
self.print_lock = threading.Lock()
def _validate_version(self):
"""Validate clang-format is the expected version."""
cf_version = callo([self.path, "--version"])
if CLANG_FORMAT_VERSION in cf_version:
return True
print(
"WARNING: clang-format with incorrect version found at "
+ self.path
+ " version: "
+ cf_version
)
return False
def _lint(self, file_name, print_diff):
"""Check the specified file has the correct format."""
with open(file_name, "rb") as original_text:
original_file = original_text.read().decode("utf-8")
original_text.seek(0)
# Get formatted file as clang-format would format the file
formatted_file = callo(
[
self.path,
"--assume-filename="
+ (file_name if not file_name.endswith(".h") else file_name + "pp"),
"--style=file",
],
stdin=original_text,
)
if original_file != formatted_file:
if print_diff:
original_lines = original_file.splitlines()
formatted_lines = formatted_file.splitlines()
result = difflib.unified_diff(original_lines, formatted_lines)
# Take a lock to ensure diffs do not get mixed when printed to the screen
with self.print_lock:
print("ERROR: Found diff for " + file_name)
print("To fix formatting errors, run `buildscripts/clang_format.py format`")
for line in result:
print(line.rstrip())
return False
return True
def lint(self, file_name):
"""Check the specified file has the correct format."""
return self._lint(file_name, print_diff=True)
def format(self, file_name):
"""Update the format of the specified file."""
if self._lint(file_name, print_diff=False):
return True
# Update the file with clang-format
# We have to tell `clang-format` to format on standard input due to its file type
# determiner. `--assume-filename` doesn't work directly on files, but only on standard
# input. Thus we have to open the file as the subprocess's standard input. Then we record
# that formatted standard output back into the file. We can't use the `-i` option, due to
# the fact that `clang-format` believes that many of our C++ headers are Objective-C code.
formatted = True
with open(file_name, "rb") as source_stream:
try:
reformatted_text = subprocess.check_output(
[
self.path,
"--assume-filename="
+ (file_name if not file_name.endswith(".h") else file_name + "pp"),
"--style=file",
],
stdin=source_stream,
)
except subprocess.CalledProcessError:
formatted = False
if formatted:
with open(file_name, "wb") as output_stream:
output_stream.write(reformatted_text)
# Version 3.8 generates files like foo.cpp~RF83372177.TMP when it formats foo.cpp
# on Windows, we must clean these up
if sys.platform == "win32":
glob_pattern = file_name + "*.TMP"
for fglob in glob.glob(glob_pattern):
os.unlink(fglob)
return formatted
FILES_RE = re.compile("\\.(h|hpp|ipp|cpp)$")
TPL_FILES_RE = re.compile("\\.tpl\\.")
def is_interesting_file(file_name):
"""Return true if this file should be checked."""
return (
(
file_name.startswith("src")
and not file_name.startswith("src/third_party/")
and not file_name.startswith("src/mongo/gotools/")
and not file_name.startswith("src/mongo/db/modules/enterprise/src/streams/third_party")
and not file_name.startswith("src/streams/third_party")
)
and FILES_RE.search(file_name)
and not TPL_FILES_RE.search(file_name)
)
def get_list_from_lines(lines):
"""Convert a string containing a series of lines into a list of strings."""
return [line.rstrip() for line in lines.splitlines()]
def _get_build_dir():
"""Return the location of the default clang cache directory."""
return os.path.join(git.get_base_dir(), ".clang_format_cache")
def _lint_files(clang_format, files):
"""Lint a list of files with clang-format."""
clang_format = ClangFormat(clang_format, _get_build_dir())
lint_clean = parallel.parallel_process([os.path.abspath(f) for f in files], clang_format.lint)
if not lint_clean:
print("ERROR: Source code does not match required source formatting style")
sys.exit(1)
def lint_patch(clang_format, infile):
"""Lint patch command entry point."""
files = git.get_files_to_check_from_patch(infile, is_interesting_file)
# Patch may have files that we do not want to check which is fine
if files:
_lint_files(clang_format, files)
def lint_git_diff(clang_format):
"""
Lint the files that have changes since the last git commit.
:param clang_format: Path to clang_format command.
"""
files = gather_changed_files_for_lint(is_interesting_file)
if files:
_lint_files(clang_format, files)
def lint(clang_format):
"""Lint files command entry point."""
files = git.get_files_to_check([], is_interesting_file)
_lint_files(clang_format, files)
return True
def lint_all(clang_format):
"""Lint files command entry point based on working tree."""
files = git.get_files_to_check_working_tree(is_interesting_file)
_lint_files(clang_format, files)
return True
def _format_files(clang_format, files):
"""Format a list of files with clang-format."""
clang_format = ClangFormat(clang_format, _get_build_dir())
format_clean = parallel.parallel_process(
[os.path.abspath(f) for f in files], clang_format.format
)
if not format_clean:
print("ERROR: failed to format files")
sys.exit(1)
def format_func(clang_format):
"""Format files command entry point."""
files = git.get_files_to_check([], is_interesting_file)
_format_files(clang_format, files)
def format_one(clang_format, filename):
"""Format file command entry point."""
_format_files(clang_format, [filename])
def format_my_func(clang_format, origin_branch):
"""My Format files command entry point."""
files = git.get_my_files_to_check(is_interesting_file, origin_branch)
files = [f for f in files if os.path.exists(f)]
_format_files(clang_format, files)
def reformat_branch(clang_format, commit_prior_to_reformat, commit_after_reformat):
"""Reformat a branch made before a clang-format run."""
clang_format = ClangFormat(clang_format, _get_build_dir())
if os.getcwd() != git.get_base_dir():
raise ValueError("reformat-branch must be run from the repo root")
if not os.path.exists("buildscripts/clang_format.py"):
raise ValueError("reformat-branch is only supported in the mongo repo")
repo = git.Repo(git.get_base_dir())
# Validate that user passes valid commits
if not repo.is_commit(commit_prior_to_reformat):
raise ValueError(
"Commit Prior to Reformat '%s' is not a valid commit in this repo"
% commit_prior_to_reformat
)
if not repo.is_commit(commit_after_reformat):
raise ValueError(
"Commit After Reformat '%s' is not a valid commit in this repo" % commit_after_reformat
)
if not repo.is_ancestor(commit_prior_to_reformat, commit_after_reformat):
raise ValueError(
(
"Commit Prior to Reformat '%s' is not a valid ancestor of Commit After"
+ " Reformat '%s' in this repo"
)
% (commit_prior_to_reformat, commit_after_reformat)
)
# Validate the user is on a local branch that has the right merge base
if repo.is_detached():
raise ValueError("You must not run this script in a detached HEAD state")
# Validate the user has no pending changes
if repo.is_working_tree_dirty():
raise ValueError(
"Your working tree has pending changes. You must have a clean working tree before proceeding."
)
merge_base = repo.get_merge_base(["HEAD", commit_prior_to_reformat])
if not merge_base == commit_prior_to_reformat:
raise ValueError(
"Please rebase to '%s' and resolve all conflicts before running this script"
% (commit_prior_to_reformat)
)
# We assume the target branch is master, it could be a different branch if needed for testing
merge_base = repo.get_merge_base(["HEAD", "master"])
if not merge_base == commit_prior_to_reformat:
raise ValueError(
"This branch appears to already have advanced too far through the merge process"
)
# Everything looks good so lets start going through all the commits
branch_name = repo.get_branch_name()
new_branch = "%s-reformatted" % branch_name
if repo.does_branch_exist(new_branch):
raise ValueError(
"The branch '%s' already exists. Please delete the branch '%s', or rename the current branch."
% (new_branch, new_branch)
)
commits = get_list_from_lines(
repo.git_log(
[
"--reverse",
"--no-show-signature",
"--pretty=format:%H",
"%s..HEAD" % commit_prior_to_reformat,
]
)
)
previous_commit_base = commit_after_reformat
# Go through all the commits the user made on the local branch and migrate to a new branch
# that is based on post_reformat commits instead
for commit_hash in commits:
repo.git_checkout(["--quiet", commit_hash])
deleted_files = []
# Format each of the files by checking out just a single commit from the user's branch
commit_files = get_list_from_lines(repo.git_diff(["HEAD~", "--name-only"]))
for commit_file in commit_files:
# Format each file needed if it was not deleted
if not os.path.exists(commit_file):
print(
"Skipping file '%s' since it has been deleted in commit '%s'"
% (commit_file, commit_hash)
)
deleted_files.append(commit_file)
continue
if is_interesting_file(commit_file):
clang_format.format(commit_file)
else:
print(
"Skipping file '%s' since it is not a file clang_format should format"
% commit_file
)
# Check if anything needed reformatting, and if so amend the commit
if not repo.is_working_tree_dirty():
print("Commit %s needed no reformatting" % commit_hash)
else:
repo.git_commit(["--all", "--amend", "--no-edit"])
# Rebase our new commit on top the post-reformat commit
previous_commit = repo.git_rev_parse(["HEAD"])
# Checkout the new branch with the reformatted commits
# Note: we will not name as a branch until we are done with all commits on the local branch
repo.git_checkout(["--quiet", previous_commit_base])
# Copy each file from the reformatted commit on top of the post reformat
diff_files = get_list_from_lines(
repo.git_diff(["%s~..%s" % (previous_commit, previous_commit), "--name-only"])
)
for diff_file in diff_files:
# If the file was deleted in the commit we are reformatting, we need to delete it again
if diff_file in deleted_files:
repo.git_rm(["--ignore-unmatch", diff_file])
continue
# The file has been added or modified, continue as normal
file_contents = repo.git_show(["%s:%s" % (previous_commit, diff_file)])
root_dir = os.path.dirname(diff_file)
if root_dir and not os.path.exists(root_dir):
os.makedirs(root_dir)
with open(diff_file, "w+", encoding="utf-8") as new_file:
new_file.write(file_contents)
repo.git_add([diff_file])
# Create a new commit onto clang-formatted branch
repo.git_commit(["--reuse-message=%s" % previous_commit, "--no-gpg-sign", "--allow-empty"])
previous_commit_base = repo.git_rev_parse(["HEAD"])
# Create a new branch to mark the hashes we have been using
repo.git_checkout(["-b", new_branch])
print("reformat-branch is done running.\n")
print(
"A copy of your branch has been made named '%s', and formatted with clang-format.\n"
% new_branch
)
print("The original branch has been left unchanged.")
print("The next step is to rebase the new branch on 'master'.")
def usage():
"""Print usage."""
print(
"clang_format.py supports 6 commands (lint, lint-all, lint-patch, format, format-my, reformat-branch)."
)
print("\nformat-my <origin branch>")
print(" <origin branch> - upstream branch to compare against")
def main():
"""Execute Main entry point."""
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
structlog.configure(logger_factory=structlog.stdlib.LoggerFactory())
parser = OptionParser()
parser.add_option("-c", "--clang-format", type="string", dest="clang_format")
(options, args) = parser.parse_args(args=sys.argv)
if len(args) > 1:
command = args[1]
if command == "lint":
lint(options.clang_format)
elif command == "lint-all":
lint_all(options.clang_format)
elif command == "lint-patch":
lint_patch(options.clang_format, args[2:])
elif command == "lint-git-diff":
lint_git_diff(options.clang_format)
elif command == "format":
format_func(options.clang_format)
elif command == "format-one":
format_one(options.clang_format, args[2])
elif command == "format-my":
format_my_func(options.clang_format, args[2] if len(args) > 2 else "origin/master")
elif command == "reformat-branch":
if len(args) < 3:
print(
"ERROR: reformat-branch takes two parameters: commit_prior_to_reformat commit_after_reformat"
)
return
reformat_branch(options.clang_format, args[2], args[3])
else:
usage()
else:
usage()
if __name__ == "__main__":
main()

7
buildscripts/s3_binary/hashes.py
View File

@ -23,4 +23,11 @@ S3_SHA256_HASHES = {
"https://mdb-build-public.s3.amazonaws.com/fd-binaries/v10.3.0/fd-linux-amd64": "9f48273b6c780a5f4f084ef30bc67d98cbd7d10c55c4605cf3a6ee29b741af87", "https://mdb-build-public.s3.amazonaws.com/fd-binaries/v10.3.0/fd-linux-amd64": "9f48273b6c780a5f4f084ef30bc67d98cbd7d10c55c4605cf3a6ee29b741af87",
"https://mdb-build-public.s3.amazonaws.com/fd-binaries/v10.3.0/fd-linux-arm64": "d0fc407937b8a8aec44f3a80b4a08219ba61dde234590358abb168b44478d493", "https://mdb-build-public.s3.amazonaws.com/fd-binaries/v10.3.0/fd-linux-arm64": "d0fc407937b8a8aec44f3a80b4a08219ba61dde234590358abb168b44478d493",
"https://mdb-build-public.s3.amazonaws.com/fd-binaries/v10.3.0/fd-windows-amd64.exe": "fd3d4853da7a319a604e1cb03ede88cbf584edd12b89a0991871fb4d9cd3ba5b", "https://mdb-build-public.s3.amazonaws.com/fd-binaries/v10.3.0/fd-windows-amd64.exe": "fd3d4853da7a319a604e1cb03ede88cbf584edd12b89a0991871fb4d9cd3ba5b",
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_darwin_x64.gz": "30eb3ed0ef541be6eea051a9097049f4b91bb565f5a0faf59ba317c061e3c3e6",
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_linux_arm64.gz": "b047e429f6f455febc970c9f59c75a92986cb638456238d71716fe2810b9177f",
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_linux_s390x.gz": "ae3c989c0a867bec6ba27d056019039be544d80cf154f4bdaa065bb9e825675c",
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_linux_x64.gz": "184065d8e70cfa3f98eae399b2444688775133733e4c4daa7a32ffdaa8602faa",
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_rhel8_ppc64le.gz": "ad0bc6188223abc115a653945ada288ac4f6232ec7226b975bb1efda8b898095",
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_rhel9_ppc64le.gz": "73805acc4786df9b414b1b28ce40680cd6c9a9a0fb58712928711ee6786c939d",
"https://mdb-build-public.s3.amazonaws.com/db-contrib-tool-binaries/v2.0.1/db-contrib-tool_v2.0.1_windows_x64.gz": "f138b09612d143fc67c8cc8efb8cfbae13ce50cf8049a7d802c5cd262176346a",
} }

10
buildscripts/smoke_tests/smoke_tests.py
View File

@ -372,16 +372,6 @@ def run_smoke_tests(
) )
formatters = [ formatters = [
runner.command(
name="clang format",
args=[
MONGO_PYTHON_INTERPRETER,
ROOT.joinpath("buildscripts", "clang_format.py"),
"format-my",
upstream_branch,
],
log_file="clang_format.log",
),
runner.command( runner.command(
name="misc. lint", name="misc. lint",
args=[ args=[

10
docs/cpp_style.md
View File

@ -1003,17 +1003,11 @@ python buildscripts/eslint.py fix # auto-fix js code
All code changes must be formatted by All code changes must be formatted by
[clang-format](http://clang.llvm.org/docs/ClangFormat.html) before they are [clang-format](http://clang.llvm.org/docs/ClangFormat.html) before they are
checked in. Use `buildscripts/clang_format.py` to reformat C++ and JS code. checked in. Use `bazel run format` to reformat C++ and JS code.
Clang-format is a C/C++ & JS code formatting tool that uses the config files Clang-format is a C/C++ & JS code formatting tool that uses the config files
located at `src/mongo/.clang-format` and `jstests/.clang-format` to control the located at `src/mongo/.clang-format` and `jstests/.clang-format` to control the
format of the code. The version and configuration of clang-format is selected by format of the code. The version and configuration of clang-format is selected by
`buildscripts/clang_format.py`. `bazel run format`.
```sh
python buildscripts/clang_format.py lint # check
python buildscripts/clang_format.py format # format all code
python buildscripts/clang_format.py format-my # reformat only changed files
```
Plugins are available for most editors that will automatically run clang-format Plugins are available for most editors that will automatically run clang-format
on file save. on file save.

11
docs/linting.md
View File

@ -2,17 +2,6 @@
## C++ Linters ## C++ Linters
### `clang-format`
The `buildscripts/clang_format.py` wrapper script runs the `clang-format` linter. You can see the
usage message for the wrapper by running `buildscripts/clang_format.py --help`.
Ex: `buildscripts/clang_format.py lint`
| Linter | Configuration File(s) | Help Command | Documentation |
| -------------- | --------------------- | --------------------- | -------------------------------------------------------------------------------------------- |
| `clang-format` | `.clang-format` | `clang-format --help` | [https://clang.llvm.org/docs/ClangFormat.html](https://clang.llvm.org/docs/ClangFormat.html) |
### `clang-tidy` ### `clang-tidy`
The `buildscripts/clang_tidy.py` shell script runs the `clang-tidy` linter. In order to run The `buildscripts/clang_tidy.py` shell script runs the `clang-tidy` linter. In order to run

17
evergreen/BUILD.bazel
View File

@ -1,3 +1,5 @@
load("@poetry//:dependencies.bzl", "dependency")
package(default_visibility = ["//visibility:public"]) package(default_visibility = ["//visibility:public"])
py_binary( py_binary(
@ -563,3 +565,18 @@ py_library(
srcs = glob(["*.py"]), srcs = glob(["*.py"]),
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
py_binary(
name = "download_db_contrib_tool",
srcs = ["download_db_contrib_tool.py"],
main = "download_db_contrib_tool.py",
visibility = ["//visibility:public"],
deps = [
dependency(
"pyyaml",
group = "core",
),
"//buildscripts/s3_binary:download",
"//buildscripts/util",
],
)

19
evergreen/download_db_contrib_tool.py
View File

@ -5,12 +5,11 @@ import platform
import re import re
import stat import stat
import sys import sys
from urllib.request import urlretrieve
import retry
mongo_path = pathlib.Path(__file__).parents[1] mongo_path = pathlib.Path(__file__).parents[1]
sys.path.append(str(mongo_path)) sys.path.append(str(mongo_path))
from buildscripts.s3_binary.download import download_s3_binary
from buildscripts.util.expansions import get_expansion from buildscripts.util.expansions import get_expansion
DB_CONTRIB_TOOL_VERSION = "v2.0.1" DB_CONTRIB_TOOL_VERSION = "v2.0.1"
@ -50,16 +49,18 @@ def get_binary_name() -> str:
return binary_name return binary_name
@retry.retry(tries=3, delay=3)
def download_binary(url: str, path: str) -> None:
urlretrieve(url, path)
def main() -> int: def main() -> int:
os.chdir(os.environ.get("BUILD_WORKSPACE_DIRECTORY", "."))
binary_name = get_binary_name() binary_name = get_binary_name()
gz_name = f"{binary_name}.gz" gz_name = f"{binary_name}.gz"
binary_url = f"{RELEASE_URL}{gz_name}" binary_url = f"{RELEASE_URL}{gz_name}"
download_binary(binary_url, gz_name) if not download_s3_binary(
s3_path=binary_url,
local_path=gz_name,
):
print("Failed to download db-contrib-tool binary")
return 1
# extract the binary # extract the binary
with gzip.open(gz_name, "rb") as fin: with gzip.open(gz_name, "rb") as fin:
with open(binary_name, "wb") as fout: with open(binary_name, "wb") as fout:

2
modules_poc/private_headers.py
View File

@ -280,7 +280,7 @@ def main(
print(f"Modified {len(idl_files_modified)} IDL files to add `mod_visibility: private`") print(f"Modified {len(idl_files_modified)} IDL files to add `mod_visibility: private`")
for file_path in idl_files_modified: for file_path in idl_files_modified:
print(f" {file_path}") print(f" {file_path}")
print("Please run 'buildscripts/clang_format.py format-my' to clean up changes to headers") print("Please run 'bazel run format' to clean up changes to headers")
else: else:
print( print(
f"\nDry run complete. Would modify {len(headers_modified) + len(idl_files_modified)} files." f"\nDry run complete. Would modify {len(headers_modified) + len(idl_files_modified)} files."