mirror of https://github.com/mongodb/mongo
SERVER-99404 improve wrapper module install detection (#31181)
GitOrigin-RevId: 01d74e5f834fcbd692051eb2ea279e955e22244b
This commit is contained in:
Daniel Moody
MongoDB Bot
parent
520f729842
commit
20551fbbb9
|
|
@ -1,16 +1,17 @@
|
|||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
sys.path.append(".")
|
||||
REPO_ROOT = os.path.join(os.path.dirname(__file__), os.path.pardir)
|
||||
sys.path.append(REPO_ROOT)
|
||||
|
||||
from buildscripts.engflow_auth import setup_auth
|
||||
from buildscripts.install_bazel import install_bazel
|
||||
|
||||
# do not print to stdout as that is used for arg modifications
|
||||
orig_stdout = sys.stdout
|
||||
# This script should be careful not to disrupt automatic mechanism which
|
||||
# may be expecting certain stdout, always print to stderr.
|
||||
sys.stdout = sys.stderr
|
||||
|
||||
if (
|
||||
|
|
@ -37,7 +38,72 @@ class DuplicateSourceNames(Exception):
|
|||
wrapper_debug(f"wrapper hook script is using {sys.executable}")
|
||||
|
||||
|
||||
def search_for_modules(deps, deps_installed, deps_needed, lockfile_changed=False):
|
||||
bazel_out_dir = os.path.join(REPO_ROOT, "bazel-out")
|
||||
if os.path.exists(bazel_out_dir):
|
||||
for dep in deps:
|
||||
found = False
|
||||
for entry in os.listdir(bazel_out_dir):
|
||||
if os.path.exists(f"{bazel_out_dir}/{entry}/bin/external/poetry/{dep}"):
|
||||
if not lockfile_changed:
|
||||
deps_installed.append(dep)
|
||||
sys.path.append(f"{bazel_out_dir}/{entry}/bin/external/poetry/{dep}")
|
||||
found = True
|
||||
break
|
||||
else:
|
||||
os.chmod(f"{bazel_out_dir}/{entry}/bin/external/poetry/{dep}", 0o777)
|
||||
for root, dirs, files in os.walk(
|
||||
f"{bazel_out_dir}/{entry}/bin/external/poetry/{dep}"
|
||||
):
|
||||
for somedir in dirs:
|
||||
os.chmod(os.path.join(root, somedir), 0o777)
|
||||
for file in files:
|
||||
os.chmod(os.path.join(root, file), 0o777)
|
||||
shutil.rmtree(f"{bazel_out_dir}/{entry}/bin/external/poetry/{dep}")
|
||||
if not found:
|
||||
deps_needed.append(dep)
|
||||
|
||||
|
||||
def install_modules(bazel):
|
||||
need_to_install = False
|
||||
pwd_hash = hashlib.md5(os.path.abspath(REPO_ROOT).encode()).hexdigest()
|
||||
lockfile_hash_file = os.path.join(tempfile.gettempdir(), f"{pwd_hash}_lockfile_hash")
|
||||
with open(os.path.join(REPO_ROOT, "poetry.lock"), "rb") as f:
|
||||
current_hash = hashlib.md5(f.read()).hexdigest()
|
||||
|
||||
old_hash = None
|
||||
if os.path.exists(lockfile_hash_file):
|
||||
with open(lockfile_hash_file) as f:
|
||||
old_hash = f.read()
|
||||
else:
|
||||
with open(lockfile_hash_file, "w") as f:
|
||||
f.write(current_hash)
|
||||
|
||||
deps = ["retry"]
|
||||
deps_installed = []
|
||||
deps_needed = []
|
||||
search_for_modules(deps, deps_installed, deps_needed, lockfile_changed=old_hash != current_hash)
|
||||
|
||||
if deps_needed:
|
||||
need_to_install = True
|
||||
|
||||
if old_hash != current_hash:
|
||||
need_to_install = True
|
||||
deps_needed = deps
|
||||
|
||||
if need_to_install:
|
||||
subprocess.run(
|
||||
[bazel, "build", "--config=local"] + ["@poetry//:install_" + dep for dep in deps_needed]
|
||||
)
|
||||
deps_missing = []
|
||||
search_for_modules(deps_needed, deps_installed, deps_missing)
|
||||
if deps_missing:
|
||||
raise Exception(f"Failed to install python deps {deps_missing}")
|
||||
|
||||
|
||||
def get_buildozer_output(autocomplete_query):
|
||||
from buildscripts.install_bazel import install_bazel
|
||||
|
||||
buildozer = shutil.which("buildozer")
|
||||
if not buildozer:
|
||||
buildozer = os.path.expanduser("~/.local/bin/buildozer")
|
||||
|
|
@ -65,6 +131,8 @@ def get_buildozer_output(autocomplete_query):
|
|||
|
||||
def engflow_auth(args):
|
||||
start = time.time()
|
||||
from buildscripts.engflow_auth import setup_auth
|
||||
|
||||
args_str = " ".join(args)
|
||||
if (
|
||||
"--config=local" not in args_str
|
||||
|
|
@ -229,10 +297,18 @@ def test_runner_interface(args, autocomplete_query, get_buildozer_output=get_bui
|
|||
return new_args
|
||||
|
||||
|
||||
engflow_auth(sys.argv)
|
||||
def main():
|
||||
install_modules(sys.argv[1])
|
||||
|
||||
args = test_runner_interface(
|
||||
sys.argv, autocomplete_query=os.environ.get("MONGO_AUTOCOMPLETE_QUERY") == "1"
|
||||
)
|
||||
engflow_auth(sys.argv)
|
||||
|
||||
print(" ".join(args), file=orig_stdout)
|
||||
args = test_runner_interface(
|
||||
sys.argv[1:], autocomplete_query=os.environ.get("MONGO_AUTOCOMPLETE_QUERY") == "1"
|
||||
)
|
||||
os.chmod(os.environ.get("MONGO_BAZEL_WRAPPER_ARGS"), 0o644)
|
||||
with open(os.environ.get("MONGO_BAZEL_WRAPPER_ARGS"), "w") as f:
|
||||
f.write(" ".join(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
36
tools/bazel
36
tools/bazel
|
|
@ -8,8 +8,9 @@ set -e
|
|||
# WARNING : If you run //:compiledb target, you can not print to stdout in this file as it will fail with
|
||||
# "Bazel aquery failed." because it is reading this files stdout as aquery output
|
||||
|
||||
REPO_ROOT=$(dirname $(dirname $(realpath "$0")))
|
||||
bazel_real="$BAZEL_REAL"
|
||||
echo $BAZEL_REAL > ".bazel_real"
|
||||
echo $BAZEL_REAL > "$REPO_ROOT/.bazel_real"
|
||||
bazelrc_xcode_lines=()
|
||||
|
||||
|
||||
|
|
@ -67,15 +68,14 @@ if [[ $OSTYPE == darwin* ]]; then
|
|||
bazelrc_lines+=("common --repo_env=DEVELOPER_DIR=$xcode_path") >&2
|
||||
fi
|
||||
|
||||
printf '%s\n' "${bazelrc_xcode_lines[@]}" > .bazelrc.xcode
|
||||
printf '%s\n' "${bazelrc_xcode_lines[@]}" > $REPO_ROOT/.bazelrc.xcode
|
||||
|
||||
echo "common --//bazel/config:running_through_bazelisk" > .bazelrc.bazelisk
|
||||
echo "common --//bazel/config:running_through_bazelisk" > $REPO_ROOT/.bazelrc.bazelisk
|
||||
|
||||
if [[ $MONGO_BAZEL_WRAPPER_DEBUG == 1 ]]; then
|
||||
wrapper_start_time="$(date -u +%s.%N)"
|
||||
fi
|
||||
|
||||
cur_dir="${PWD##*/}"
|
||||
|
||||
|
||||
if [[ "$OSTYPE" == "linux"* ]]; then
|
||||
os="linux"
|
||||
|
|
@ -97,17 +97,6 @@ else
|
|||
ARCH="x86_64"
|
||||
fi
|
||||
|
||||
python="bazel-$cur_dir/external/py_${os}_${ARCH}/dist/bin/python3"
|
||||
|
||||
wrapper_deps=("retry")
|
||||
declare -a wrapper_deps_to_install=()
|
||||
declare -a python_path=()
|
||||
for dep in ${wrapper_deps[@]}; do
|
||||
python_path+=("bazel-bin/external/poetry/$dep")
|
||||
if [ ! -d bazel-bin/external/poetry/$dep ] || [ ! -d bazel-$cur_dir/external/py_${os}_${ARCH}/dist/bin ] ; then
|
||||
wrapper_deps_to_install+=(@poetry//:install_$dep)
|
||||
fi
|
||||
done
|
||||
|
||||
declare -a cert_locs=()
|
||||
cert_locs+=("/etc/ssl/certs/ca-certificates.crt") # Debian/Ubuntu/Gentoo etc.
|
||||
|
|
@ -125,9 +114,11 @@ for cert in ${cert_locs[@]}; do
|
|||
fi
|
||||
done
|
||||
|
||||
if [[ ${#wrapper_deps_to_install[@]} != 0 ]]; then
|
||||
cur_dir=$(basename $REPO_ROOT)
|
||||
python="$REPO_ROOT/bazel-$cur_dir/external/py_${os}_${ARCH}/dist/bin/python3"
|
||||
if [ ! -f $python ]; then
|
||||
>&2 echo "python prereq missing, using bazel to install python..."
|
||||
>&2 $bazel_real build --config=local $(IFS= ; echo "${wrapper_deps_to_install[*]}")
|
||||
>&2 $bazel_real build --config=local @py_${os}_${ARCH}//:all
|
||||
fi
|
||||
|
||||
autocomplete_query=0
|
||||
|
|
@ -141,8 +132,13 @@ if [[ $* == *"--noblock_for_lock query kind(\".*_test\", //:all)"* ]] || [[ $* =
|
|||
fi
|
||||
|
||||
rm -f /tmp/mongo_autocomplete_plus_targets
|
||||
python="bazel-$cur_dir/external/py_${os}_${ARCH}/dist/bin/python3"
|
||||
new_args=$(MONGO_AUTOCOMPLETE_QUERY=$autocomplete_query PYTHONPATH=$(IFS=: ; echo "${python_path[*]}") $python bazel/wrapper_hook.py "$@")
|
||||
MONGO_BAZEL_WRAPPER_ARGS=$(mktemp)
|
||||
|
||||
MONGO_BAZEL_WRAPPER_ARGS=$MONGO_BAZEL_WRAPPER_ARGS \
|
||||
MONGO_AUTOCOMPLETE_QUERY=$autocomplete_query \
|
||||
$python $REPO_ROOT/bazel/wrapper_hook.py $bazel_real "$@"
|
||||
|
||||
new_args=$(<$MONGO_BAZEL_WRAPPER_ARGS)
|
||||
|
||||
if [[ $MONGO_BAZEL_WRAPPER_DEBUG == 1 ]] && [[ $autocomplete_query == 0 ]]; then
|
||||
wrapper_end_time="$(date -u +%s.%N)"
|
||||
|
|
|
|||
|
|
@ -3,40 +3,23 @@ setlocal EnableDelayedExpansion
|
|||
|
||||
echo common --//bazel/config:running_through_bazelisk > .bazelrc.bazelisk
|
||||
|
||||
for %%I in (.) do set cur_dir=%%~nxI
|
||||
set REPO_ROOT=%~dp0\..
|
||||
|
||||
set python="bazel-%cur_dir%\external\py_windows_x86_64\dist\python.exe"
|
||||
for %%I in (%REPO_ROOT%) do set cur_dir=%%~nxI
|
||||
|
||||
set "wrapper_deps="
|
||||
set "wrapper_deps_to_install="
|
||||
set "PYTHONPATH="
|
||||
set python="%REPO_ROOT%\bazel-%cur_dir%\external\py_windows_x86_64\dist\python.exe"
|
||||
|
||||
set "wrapper_deps=!wrapper_deps! retry"
|
||||
REM set "wrapper_deps=!wrapper_deps! example_append"
|
||||
set len=0
|
||||
(for %%d in (!wrapper_deps!) do (
|
||||
set "PYTHONPATH=%PYTHONPATH%;bazel-bin/external/poetry/%%d"
|
||||
if not exist "bazel-bin\external\poetry/%%d" (
|
||||
set /a len+=1
|
||||
set "wrapper_deps_to_install=!wrapper_deps_to_install! @poetry//:install_%%d"
|
||||
) else (
|
||||
if not exist "%python%" (
|
||||
set /a len+=1
|
||||
set "wrapper_deps_to_install=!wrapper_deps_to_install! @poetry//:install_%%d"
|
||||
)
|
||||
)
|
||||
))
|
||||
|
||||
if %len% gtr 0 (
|
||||
if not exist "%python%" (
|
||||
echo python prereq missing, using bazel to install python... 1>&2
|
||||
"%BAZEL_REAL%" build %wrapper_deps_to_install% 1>&2
|
||||
"%BAZEL_REAL%" build --config=local @py_windows_x86_64//:all 1>&2
|
||||
)
|
||||
SET STARTTIME=%TIME%
|
||||
|
||||
set "uniqueFileName=%tmp%\bat~%RANDOM%.tmp"
|
||||
%python% bazel/wrapper_hook.py %* > %uniqueFileName%
|
||||
for /f "Tokens=* Delims=" %%x in ( %uniqueFileName% ) do set "new_args=!new_args!%%x"
|
||||
del %uniqueFileName%
|
||||
set "MONGO_BAZEL_WRAPPER_ARGS=%tmp%\bat~%RANDOM%.tmp"
|
||||
echo "" > %MONGO_BAZEL_WRAPPER_ARGS%
|
||||
%python% %REPO_ROOT%/bazel/wrapper_hook.py "%BAZEL_REAL%" %*
|
||||
for /f "Tokens=* Delims=" %%x in ( %MONGO_BAZEL_WRAPPER_ARGS% ) do set "new_args=!new_args!%%x"
|
||||
del %MONGO_BAZEL_WRAPPER_ARGS%
|
||||
|
||||
REM Final Calculations
|
||||
SET ENDTIME=%TIME%
|
||||
|
|
|
|||
Loading…
Reference in New Issue