From b4486819a44b92cf462d1aca480bc54b1602e371 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 23 May 2025 14:00:21 -0500 Subject: [PATCH] SERVER-105429 make poetry requrements file (#36472) GitOrigin-RevId: c16188de078d8819b30184e34ca87baed6cf57c2 --- .github/CODEOWNERS | 1 + OWNERS.yml | 4 ++ .../base_images/workload/Dockerfile | 4 +- buildscripts/pip_requirements.py | 8 +++- buildscripts/poetry_sync.sh | 2 +- .../resmokelib/powercycle/setup/__init__.py | 11 ++++- etc/evergreen_yml_components/definitions.yml | 1 + .../tasks/compile_tasks_shared.yml | 1 + .../tasks/resmoke/non_server_teams/tasks.yml | 1 + etc/set_up_workstation.sh | 2 +- evergreen/functions/venv_setup.sh | 2 +- poetry_requirements.txt | 41 +++++++++++++++++++ 12 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 poetry_requirements.txt diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4570efbe265..a04fc927013 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -22,6 +22,7 @@ package.json @10gen/devprod-correctness @svc-auto-approve-bot pnpm-lock.yaml @10gen/devprod-correctness @svc-auto-approve-bot poetry.lock @10gen/devprod-build @10gen/devprod-correctness @svc-auto-approve-bot pyproject.toml @10gen/devprod-build @10gen/devprod-correctness @svc-auto-approve-bot +poetry_requirements.txt @10gen/devprod-build @10gen/devprod-correctness @svc-auto-approve-bot sbom.json @10gen/server-security @svc-auto-approve-bot MODULE.bazel* @10gen/devprod-build @svc-auto-approve-bot WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot diff --git a/OWNERS.yml b/OWNERS.yml index c3ecf5b4efa..7bbd7511361 100644 --- a/OWNERS.yml +++ b/OWNERS.yml @@ -58,6 +58,10 @@ filters: approvers: - 10gen/devprod-correctness - 10gen/devprod-build + - "poetry_requirements.txt": + approvers: + - 10gen/devprod-correctness + - 10gen/devprod-build - "sbom.json": approvers: - 10gen/server-security diff --git a/buildscripts/antithesis/base_images/workload/Dockerfile b/buildscripts/antithesis/base_images/workload/Dockerfile index 067078d6a19..1687b6c2cd1 100644 --- a/buildscripts/antithesis/base_images/workload/Dockerfile +++ b/buildscripts/antithesis/base_images/workload/Dockerfile @@ -75,10 +75,10 @@ RUN ./src/scripts/npm_run.sh jstestfuzz -- --help WORKDIR /mongo # Copy mongodb python deps first -COPY src/pyproject.toml src/poetry.lock ./ +COPY src/pyproject.toml src/poetry.lock src/poetry_requirements.txt ./ # Install mongodb python deps with poetry -RUN python -m pip install 'poetry==2.0.0' +RUN python -m pip install -r poetry_requirements.txt RUN python -m poetry install --no-root --sync # Add the poetry venv to the $PATH so that it's activated by default diff --git a/buildscripts/pip_requirements.py b/buildscripts/pip_requirements.py index dd3eb74d0cb..0cc56ec49d7 100644 --- a/buildscripts/pip_requirements.py +++ b/buildscripts/pip_requirements.py @@ -39,8 +39,12 @@ def verify_requirements(silent: bool = False, executable=sys.executable): # This is so the user will have an easier time diagnosing the problem try: import poetry # noqa: F401 - except ModuleNotFoundError as ex: - raiseSuggestion(ex, "'poetry==2.0.0'") + except ModuleNotFoundError: + raise MissingRequirements( + f"Detected one or more packages are out of date. " + f"Try running:\n" + f" buildscripts/poetry_sync.sh -p '{executable}'\n" + ) verbose("Checking required python packages...") diff --git a/buildscripts/poetry_sync.sh b/buildscripts/poetry_sync.sh index 64b2e12972e..2d4746a817f 100755 --- a/buildscripts/poetry_sync.sh +++ b/buildscripts/poetry_sync.sh @@ -77,7 +77,7 @@ if [[ "${allow_no_venv}" != 1 ]]; then # Exploit pip's own enforcement of virtualenv. pip_opts+=('--require-virtualenv') fi -run "${py3}" -m pip install "${pip_opts[@]}" "poetry==${poetry_version}" +run "${py3}" -m pip install "${pip_opts[@]}" -r poetry_requirements.txt run env \ PYTHON_KEYRING_BACKEND="keyring.backends.null.Keyring" \ diff --git a/buildscripts/resmokelib/powercycle/setup/__init__.py b/buildscripts/resmokelib/powercycle/setup/__init__.py index 0f9218c754f..52fccb30062 100644 --- a/buildscripts/resmokelib/powercycle/setup/__init__.py +++ b/buildscripts/resmokelib/powercycle/setup/__init__.py @@ -38,7 +38,14 @@ class SetUpEC2Instance(PowercycleCommand): # Second operation - # Copy buildscripts and mongoDB executables to the remote host. - files = ["etc", "buildscripts", "dist-test/bin", "poetry.lock", "pyproject.toml"] + files = [ + "etc", + "buildscripts", + "dist-test/bin", + "poetry.lock", + "pyproject.toml", + "poetry_requirements.txt", + ] shared_libs = "dist-test/lib" if os.path.isdir(shared_libs): @@ -63,7 +70,7 @@ class SetUpEC2Instance(PowercycleCommand): cmds = f"{cmds}; $python_loc -m venv --system-site-packages {venv}" cmds = f"{cmds}; activate=$(find {venv} -name 'activate')" cmds = f"{cmds}; . $activate" - cmds = f"{cmds}; python3 -m pip install 'poetry==2.0.0'" + cmds = f"{cmds}; pushd $remote_dir && python3 -m pip install -r poetry_requirements.txt && popd" cmds = f"{cmds}; pushd $remote_dir && python3 -m poetry install --no-root --sync && popd" self.remote_op.operation(SSHOperation.SHELL, cmds, retry=True, retry_count=retry_count) diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index b1e76ed362e..820c4c7c5bd 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -816,6 +816,7 @@ functions: - *add_git_tag - *kill_processes - *cleanup_environment + - *set_up_venv "do non-compile setup": - command: manifest.load diff --git a/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml b/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml index 9135d96b712..9af9c4bd507 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks_shared.yml @@ -193,6 +193,7 @@ tasks: - "src/src/third_party/schemastore.org/**" - "src/poetry.lock" - "src/pyproject.toml" + - "src/poetry_requirements.txt" - "src/x509/**" exclude_files: - "src/*_test.pdb" diff --git a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml index bfcef7a96a1..e957003a154 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml @@ -121,6 +121,7 @@ tasks: depends_on: [] commands: - func: "do pre workstation setup" + - func: "get engflow creds" - func: "set up then check workstation script" - <<: *gen_burn_in_task_template diff --git a/etc/set_up_workstation.sh b/etc/set_up_workstation.sh index edda75298e5..8459b367fe7 100755 --- a/etc/set_up_workstation.sh +++ b/etc/set_up_workstation.sh @@ -70,7 +70,7 @@ setup_poetry() { if command -v poetry &> /dev/null; then echo "'poetry' command exists; skipping setup" else - pipx install poetry==2.0.0 + pipx install poetry --pip-args="-r $(pwd)/poetry_requirements.txt" echo "Finished installing poetry..." fi } diff --git a/evergreen/functions/venv_setup.sh b/evergreen/functions/venv_setup.sh index 6aa2dd44207..2ad93f1232f 100644 --- a/evergreen/functions/venv_setup.sh +++ b/evergreen/functions/venv_setup.sh @@ -35,7 +35,7 @@ export POETRY_DATA_DIR="$poetry_dir/data" export POETRY_CACHE_DIR="$poetry_dir/cache" export PIP_CACHE_DIR="$poetry_dir/pip_cache" for i in {1..5}; do - $POETRY_VENV_PYTHON -m pip install "poetry==2.0.0" && RET=0 && break || RET=$? && sleep 1 + $POETRY_VENV_PYTHON -m pip install -r src/poetry_requirements.txt && RET=0 && break || RET=$? && sleep 1 echo "Python failed to install poetry, retrying..." done diff --git a/poetry_requirements.txt b/poetry_requirements.txt new file mode 100644 index 00000000000..cf072f60f57 --- /dev/null +++ b/poetry_requirements.txt @@ -0,0 +1,41 @@ +backports.tarfile~=1.2.0 +build~=1.2.2.post1 +CacheControl~=0.14.3 +certifi~=2025.4.26 +cffi~=1.17.1 +charset-normalizer~=3.4.2 +cleo~=2.1.0 +crashtest~=0.4.1 +cryptography~=44.0.2 +distlib~=0.3.9 +dulwich~=0.22.8 +fastjsonschema~=2.21.1 +filelock~=3.18.0 +idna~=3.10 +importlib_metadata~=8.7.0 +installer~=0.7.0 +jaraco.classes~=3.4.0 +jaraco.context~=6.0.1 +jaraco.functools~=4.1.0 +jeepney~=0.9.0 +keyring~=25.6.0 +more-itertools~=10.7.0 +msgpack~=1.1.0 +packaging~=25.0 +pkginfo~=1.12.1.2 +platformdirs~=4.3.8 +poetry~=2.0.0 +poetry-core~=2.0.0 +pycparser~=2.22 +pyproject_hooks~=1.2.0 +RapidFuzz~=3.13.0 +requests~=2.32.3 +requests-toolbelt~=1.0.0 +SecretStorage~=3.3.3 +shellingham~=1.5.4 +tomli~=2.2.1 +tomlkit~=0.13.2 +trove-classifiers~=2025.5.9.12 +urllib3~=2.4.0 +virtualenv~=20.31.2 +zipp~=3.21.0