diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 413a0f82108..326146e4450 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -39,13 +39,6 @@ RUN chmod +x /usr/local/bin/xdg-open-wrapper.sh && \ RUN echo "${USERNAME} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/devcontaineruser && \ chmod 0440 /etc/sudoers.d/devcontaineruser -# Persistent bash history -RUN SNIPPET="export PROMPT_COMMAND='history -a' && export HISTFILE=/commandhistory/.bash_history" \ - && mkdir /commandhistory \ - && touch /commandhistory/.bash_history \ - && chown -R $USERNAME /commandhistory \ - && echo "$SNIPPET" >> "/home/$USERNAME/.bashrc" - # Toolchain installation with SHA256 verification # Run "python3 toolchain.py generate" to update toolchain_config.env ARG TARGETPLATFORM @@ -80,35 +73,41 @@ USER $USERNAME ENV USER=${USERNAME} RUN /opt/mongodbtoolchain/revisions/*/scripts/install.sh; echo "Toolchain installation complete" -# Add MongoDB toolchain to PATH -ENV PATH="/opt/mongodbtoolchain/v5/bin:${PATH}" - -# Setup bash profile to source .bashrc -RUN echo "" >> "$HOME/.bash_profile" && \ - echo "# BEGIN Source .bashrc" >> "$HOME/.bash_profile" && \ - echo "if [[ -f ~/.bashrc ]]; then" >> "$HOME/.bash_profile" && \ - echo " source ~/.bashrc" >> "$HOME/.bash_profile" && \ - echo "fi" >> "$HOME/.bash_profile" && \ - echo "# END Source .bashrc" >> "$HOME/.bash_profile" +# Add MongoDB toolchain to PATH via system-wide profile +USER root +RUN echo 'export PATH="/opt/mongodbtoolchain/v5/bin:${PATH}"' > /etc/profile.d/02-mongodbtoolchain.sh \ + && chmod +x /etc/profile.d/02-mongodbtoolchain.sh +USER $USERNAME # Create MongoDB data directory USER root RUN mkdir -p /data/db && chown -R ${USERNAME}:${USERNAME} /data/db USER $USERNAME -# Bazel telemetry -RUN echo "common --bes_keywords=devcontainer:use=true" >> "$HOME/.bazelrc" && \ - echo "common --bes_keywords=devcontainer:image=$BASE_IMAGE" >> "$HOME/.bazelrc" && \ - echo "common --bes_keywords=devcontainer:username=$USERNAME" >> "$HOME/.bazelrc" +# Bazel telemetry - configure system-wide defaults +# These will be imported by user's .bazelrc in post-create script +USER root +RUN mkdir -p /etc/devcontainer && \ + echo "# MongoDB Devcontainer Bazel Configuration" > /etc/devcontainer/bazelrc && \ + echo "common --bes_keywords=devcontainer:use=true" >> /etc/devcontainer/bazelrc && \ + echo "common --bes_keywords=devcontainer:image=$BASE_IMAGE" >> /etc/devcontainer/bazelrc && \ + chmod 644 /etc/devcontainer/bazelrc +USER $USERNAME # Install pipx (Python package manager for tools) +# Add ~/.local/bin to PATH for pipx-installed tools +USER root +RUN echo 'export PATH="$HOME/.local/bin:${PATH}"' > /etc/profile.d/03-local-bin.sh \ + && chmod +x /etc/profile.d/03-local-bin.sh +USER $USERNAME + ENV PATH="/home/${USERNAME}/.local/bin:${PATH}" RUN /opt/mongodbtoolchain/v5/bin/python3 -m venv /tmp/pipx-venv && \ /tmp/pipx-venv/bin/python -m pip install --upgrade "pip<20.3" && \ /tmp/pipx-venv/bin/python -m pip install pipx && \ /tmp/pipx-venv/bin/pipx install pipx --python /opt/mongodbtoolchain/v5/bin/python3 --force && \ - /tmp/pipx-venv/bin/pipx ensurepath --force && \ rm -rf /tmp/pipx-venv +# Note: PATH is configured via /etc/profile.d, not ~/.bashrc, to avoid modifying home volume # Install db-contrib-tool using pipx RUN /home/${USERNAME}/.local/bin/pipx install db-contrib-tool diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 26545ff0b31..d1e1d54ec03 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,6 +11,11 @@ } }, "mounts": [ + { + "source": "mongo-dev-home", + "target": "/home", + "type": "volume" + }, { "source": "engflow_auth", "target": "/home/${localEnv:USER}/.config/engflow_auth", @@ -21,15 +26,16 @@ "target": "/home/${localEnv:USER}/.cache", "type": "volume" }, - { - "source": "mongo-bashhistory", - "target": "/commandhistory", - "type": "volume" - }, { "source": "${containerWorkspaceFolderBasename}-python3-venv", "target": "${containerWorkspaceFolder}/python3-venv", "type": "volume" + }, + { + "source": "${localEnv:HOME}/.ssh", + "target": "/home/${localEnv:USER}/.ssh", + "type": "bind", + "readonly": true } ], "containerEnv": { diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index 64c1a23934c..40b9585e67a 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -8,10 +8,28 @@ set -euo pipefail echo "Starting MongoDB development container post-create setup..." -# Step 1: Fix volume permissions +# Setup user's .bazelrc to import system-wide devcontainer config +echo "Setting up Bazel configuration..." +if [ -f "/etc/devcontainer/bazelrc" ]; then + if [ ! -f "${HOME}/.bazelrc" ]; then + # Create new .bazelrc with import + echo "# Import devcontainer system configuration" >"${HOME}/.bazelrc" + echo "try-import /etc/devcontainer/bazelrc" >>"${HOME}/.bazelrc" + echo "[OK] Created .bazelrc with system config import" + elif ! grep -q "try-import /etc/devcontainer/bazelrc" "${HOME}/.bazelrc" 2>/dev/null; then + # Add import to existing .bazelrc + echo "" >>"${HOME}/.bazelrc" + echo "# Import devcontainer system configuration" >>"${HOME}/.bazelrc" + echo "try-import /etc/devcontainer/bazelrc" >>"${HOME}/.bazelrc" + echo "[OK] Added system config import to existing .bazelrc" + else + echo "Info: .bazelrc already imports system config" + fi +fi + +# Fix volume permissions echo "Fixing volume permissions..." -sudo chown -R "$(whoami)": "${HOME}/.config/engflow_auth" || echo "Warning: Could not fix engflow_auth permissions" -sudo chown -R "$(whoami)": "${HOME}/.cache" || echo "Warning: Could not fix cache permissions" +sudo chown -R "$(whoami)": "${HOME}" || echo "Warning: Could not fix home directory permissions" sudo chown -R "$(whoami)": "${WORKSPACE_FOLDER}/python3-venv" || echo "Warning: Could not fix python3-venv permissions" sudo chown "$(whoami)": "${WORKSPACE_FOLDER}/.." || echo "Warning: Could not fix parent directory permissions" @@ -28,7 +46,7 @@ echo "Configuring git safe.directory..." git config --global --add safe.directory "${WORKSPACE_FOLDER}" || echo "Warning: Could not configure git safe.directory" echo "[OK] Volume and Git permissions fixed" -# Step 2: Configure Bazel with Docker information (one-time container setup) +# Configure Bazel with Docker information (one-time container setup) echo "Configuring Bazel with Docker server information..." # Helper function to add Bazel keyword if not already present @@ -57,7 +75,7 @@ add_bazel_keyword "docker_server_version" "${DOCKER_VERSION}" ARCH=$(uname -i 2>/dev/null || echo "unknown") add_bazel_keyword "arch" "${ARCH}" -# Step 4: Unshallow Git repository (one-time setup) +# Unshallow Git repository (one-time setup) echo "Setting up Git repository..." # Helper function to handle git shallow lock @@ -100,7 +118,7 @@ else fi fi -# Step 4: Create Python virtual environment and install dependencies +# Create Python virtual environment and install dependencies echo "Creating Python virtual environment..." venv_created=false if [ ! -d "${WORKSPACE_FOLDER}/python3-venv/bin" ]; then @@ -127,7 +145,7 @@ else echo "Info: Python virtual environment already exists" fi -# Step 5: Build clang configuration +# Build clang configuration echo "Building clang configuration..." # Backup existing .bazelrc.compiledb if it exists, then use our known-good config @@ -169,7 +187,7 @@ else echo "Warning: Failed to build clang configuration" fi -# Step 6: Setup GDB pretty printers +# Setup GDB pretty printers echo "Setting up GDB pretty printers..." cd "${WORKSPACE_FOLDER}/.." if [ -d "Boost-Pretty-Printer" ]; then @@ -189,7 +207,7 @@ else fi cd "${WORKSPACE_FOLDER}" -# Step 7: Configure automatic Python virtual environment activation +# Configure automatic Python virtual environment activation echo "Configuring automatic Python virtual environment activation..." # Helper function to add Python venv activation to shell config @@ -220,7 +238,7 @@ else echo "Warning: Python virtual environment not found at ${WORKSPACE_FOLDER}/python3-venv" fi -# Step 8: Sync Python dependencies +# Sync Python dependencies echo "Syncing Python dependencies..." if [ -f "${WORKSPACE_FOLDER}/python3-venv/bin/activate" ]; then # Only run sync if venv was just created OR if we're updating an existing one @@ -241,7 +259,7 @@ else echo "Warning: Python virtual environment not found at ${WORKSPACE_FOLDER}/python3-venv" fi -# Step 9: Install Node modules +# Install Node modules echo "Installing Node.js dependencies..." if bazel run @pnpm//:pnpm --config=local -- --dir "${WORKSPACE_FOLDER}" install; then echo "[OK] Node.js dependencies installed successfully" diff --git a/docs/devcontainer/getting-started.md b/docs/devcontainer/getting-started.md index d0ab52f4b16..8045f1a70ad 100644 --- a/docs/devcontainer/getting-started.md +++ b/docs/devcontainer/getting-started.md @@ -67,18 +67,29 @@ For Linux users, you can use Docker Engine directly. **Installation:** Follow the official guide: [docs.docker.com/engine/install](https://docs.docker.com/engine/install/) -### 2. Install Visual Studio Code +### 2. Create SSH Directory (Required) + +> **⚠️ Critical:** You **must** have a `~/.ssh` directory on your host machine before building the devcontainer. The devcontainer requires this directory to exist, regardless of whether you use SSH or HTTPS to clone the repository. + +```bash +# On your HOST machine (not inside the container) +mkdir -p ~/.ssh +``` + +If you skip this step, you'll encounter bind mount errors when trying to start the devcontainer. + +### 3. Install Visual Studio Code Download and install VS Code from [code.visualstudio.com](https://code.visualstudio.com/) -### 3. Install Dev Containers Extension +### 4. Install Dev Containers Extension 1. Open VS Code 2. Go to Extensions (⌘/Ctrl+Shift+X) 3. Search for "Dev Containers" 4. Install the [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension by Microsoft -### 4. Configure SSH Keys (Recommended) +### 5. Configure SSH Keys (Recommended) To clone the repository using SSH (recommended for contributors), you'll need SSH keys configured with GitHub. diff --git a/docs/devcontainer/troubleshooting.md b/docs/devcontainer/troubleshooting.md index 79d9eca35c2..4776d509352 100644 --- a/docs/devcontainer/troubleshooting.md +++ b/docs/devcontainer/troubleshooting.md @@ -32,6 +32,54 @@ Restart VSCode. If you install Rancher Desktop while you already have VSCode ope ## Container Build Issues +### Build Fails with SSH Bind Mount Error + +**Symptoms:** + +``` +Error response from daemon: invalid mount config for type "bind": bind source path does not exist: /Users/username/.ssh +``` + +Or on macOS/Linux systems using certain Docker providers: + +``` +Error response from daemon: invalid mount config for type "bind": bind source path does not exist: /socket_mnt/... +``` + +**Root Cause:** + +The devcontainer configuration mounts your `~/.ssh` directory to enable Git operations over SSH. If this directory doesn't exist on your host machine, the container fails to start. **This directory is required even if you plan to use HTTPS instead of SSH for cloning.** + +**Solutions:** + +1. **Create the .ssh directory on your host machine:** + + ```bash + # On your HOST machine (not in container) + mkdir -p ~/.ssh + ``` + +2. **Rebuild the container:** + + - Command Palette → "Dev Containers: Rebuild Container" + +**Note on SSH Agent Forwarding:** + +SSH agent forwarding behavior varies by Docker provider on macOS: + +- **Docker Desktop**: Automatic SSH agent forwarding built-in +- **OrbStack**: Automatic SSH agent forwarding built-in +- **Rancher Desktop**: + - With dockerd runtime: Automatic agent forwarding + - With containerd runtime: Agent forwarding requires additional setup + +To use SSH agent forwarding, ensure your SSH keys are added to your host's SSH agent before starting the container: + +```bash +ssh-add ~/.ssh/id_ed25519 # or your key name +ssh-add -l # verify keys are loaded +``` + ### Build Fails with "No Space Left on Device" **Symptoms:** diff --git a/etc/evergreen_yml_components/variants/misc/misc.yml b/etc/evergreen_yml_components/variants/misc/misc.yml index 9e574f1452a..bff7337bd6c 100644 --- a/etc/evergreen_yml_components/variants/misc/misc.yml +++ b/etc/evergreen_yml_components/variants/misc/misc.yml @@ -89,6 +89,6 @@ buildvariants: - "evergreen/devcontainer*.sh" - "poetry_requirements.txt" run_on: - - ubuntu2204-small + - ubuntu2204-large tasks: - name: devcontainer_test diff --git a/evergreen/devcontainer_cli_setup.sh b/evergreen/devcontainer_cli_setup.sh index 33b588e2219..740646215c8 100755 --- a/evergreen/devcontainer_cli_setup.sh +++ b/evergreen/devcontainer_cli_setup.sh @@ -26,11 +26,15 @@ fi echo "npm version:" npm --version -# Install devcontainer CLI locally in task workdir +# Install devcontainer CLI to temp directory echo "" echo "Installing @devcontainers/cli..." -INSTALL_DIR="$PWD/.devcontainer-cli" -mkdir -p "$INSTALL_DIR" +# Use CLI_INSTALL_DIR set by caller +if [ -z "${CLI_INSTALL_DIR:-}" ]; then + echo "ERROR: CLI_INSTALL_DIR not set by caller" + exit 1 +fi +INSTALL_DIR="$CLI_INSTALL_DIR" npm install -g --prefix "$INSTALL_DIR" @devcontainers/cli # Add to PATH for this script and subsequent commands diff --git a/evergreen/devcontainer_test.sh b/evergreen/devcontainer_test.sh index 0f602701640..fa126d2ec9c 100755 --- a/evergreen/devcontainer_test.sh +++ b/evergreen/devcontainer_test.sh @@ -13,6 +13,19 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" cd "$REPO_ROOT" +# Create temp directory for devcontainer CLI installation +CLI_INSTALL_DIR=$(mktemp -d) +export CLI_INSTALL_DIR + +# Set up cleanup trap to remove temp directory on exit +cleanup() { + if [ -d "$CLI_INSTALL_DIR" ]; then + echo "Cleaning up devcontainer CLI installation..." + rm -rf "$CLI_INSTALL_DIR" + fi +} +trap cleanup EXIT + # Set a non-conflicting username for CI # In CI, USER is often "ubuntu" which conflicts with system users # Use a distinct name that won't conflict @@ -37,8 +50,8 @@ if ! command -v devcontainer &>/dev/null; then echo "Installing devcontainer CLI..." bash "$SCRIPT_DIR/devcontainer_cli_setup.sh" - # Add CLI to PATH (installed locally by cli_setup.sh) - export PATH="$PWD/.devcontainer-cli/bin:$PATH" + # Add CLI to PATH (installed by cli_setup.sh) + export PATH="${CLI_INSTALL_DIR}/bin:$PATH" fi echo "Using devcontainer CLI version:" @@ -76,30 +89,35 @@ echo "Container ID: $CONTAINER_ID" echo "" echo "=== Testing inside devcontainer ===" -# Run commands inside the container using devcontainer exec +# Helper function to run commands inside the container +# Pass CI=true to all exec commands so engflow_auth wrapper skips credential helper setup # Pass the same --id-label so it can find the container we created +devcontainer_run() { + devcontainer exec --workspace-folder . --id-label "test-id=${TEST_ID}" --remote-env CI=true "$@" +} + echo "Checking GCC version..." -devcontainer exec --workspace-folder . --id-label "test-id=${TEST_ID}" gcc --version +devcontainer_run gcc --version echo "" echo "Checking Python version..." -devcontainer exec --workspace-folder . --id-label "test-id=${TEST_ID}" python3 --version +devcontainer_run python3 --version echo "" echo "Checking Python venv..." -devcontainer exec --workspace-folder . --id-label "test-id=${TEST_ID}" bash -c "source python3-venv/bin/activate && python --version" +devcontainer_run bash -c "source python3-venv/bin/activate && python --version" echo "" echo "Checking Bazel..." -devcontainer exec --workspace-folder . --id-label "test-id=${TEST_ID}" bazel --version +devcontainer_run bazel --version echo "" echo "Checking Git..." -devcontainer exec --workspace-folder . --id-label "test-id=${TEST_ID}" git --version +devcontainer_run git --version echo "" echo "Checking clangd configuration..." -if ! devcontainer exec --workspace-folder . --id-label "test-id=${TEST_ID}" test -f compile_commands.json; then +if ! devcontainer_run test -f compile_commands.json; then echo "ERROR: compile_commands.json not found - clangd setup failed" exit 1 fi @@ -107,12 +125,42 @@ echo "✓ compile_commands.json exists" echo "" echo "Checking .clang-tidy configuration..." -if ! devcontainer exec --workspace-folder . --id-label "test-id=${TEST_ID}" test -f .clang-tidy; then +if ! devcontainer_run test -f .clang-tidy; then echo "ERROR: .clang-tidy not found - clang-tidy setup failed" exit 1 fi echo "✓ .clang-tidy exists" +echo "" +echo "=== Running representative user operations ===" + +echo "" +echo "Running C++ unit test..." +devcontainer_run bazel test //src/mongo/bson:bson_test --config=local --test_output=errors + +echo "" +echo "Building IDL target (tests code generation)..." +devcontainer_run bazel build //src/mongo/bson:bson_validate --config=local + +echo "" +echo "Building install-dist-test target (necessary for resmoke below)..." +devcontainer_run bazel build install-dist-test + +echo "" +echo "Running Python test via resmoke..." +devcontainer_run python3 buildscripts/resmoke.py run --suite=core --sanityCheck + +echo "" +echo "Checking code formatting..." +devcontainer_run bazel run format + +echo "" +echo "Running linter..." +devcontainer_run bazel run lint + +echo "" +echo "✅ All representative operations passed" + echo "" echo "=== Stopping devcontainer ==="