diff --git a/BUILD.bazel b/BUILD.bazel index 40664e405e3..44fbe403c49 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -441,3 +441,21 @@ mongo_install( "//src/mongo/replay:mongor", ], ) + +sh_binary( + name = "scip-clang", + srcs = ["//bazel:run_under_working_dir.sh"], + data = ["@scip-clang//file"], + env = { + "BINARY_PATH": "$(location @scip-clang//file)", + }, +) + +sh_binary( + name = "scip-src", + srcs = ["//bazel:run_under_working_dir.sh"], + data = ["@scip-src//file"], + env = { + "BINARY_PATH": "$(location @scip-src//file)", + }, +) diff --git a/MODULE.bazel b/MODULE.bazel index 7d4ab2de556..0e2538e9c33 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -11,6 +11,8 @@ use_repo(bazel_features_deps, "bazel_features_globals", "bazel_features_version" http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + http_archive( name = "bazel_clang_tidy", integrity = "sha256-A+TGGfuHdS7vWT20eyAEiA3u4YXFtuzsJDcpTEmDMS0=", @@ -22,6 +24,30 @@ http_archive( ] * 5, ) +# SourceGraph indexer +http_file( + name = "scip-clang", + executable = True, + sha256 = "9fc7d1c309f8adee1b8663efa802efa7363ef22ee06f4711117535e406690689", + urls = [ + # Implements retry by relisting each url multiple times to be used as a failover. + # TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries + "https://github.com/sourcegraph/scip-clang/releases/download/v0.3.2/scip-clang-x86_64-linux", + ] * 5, +) + +# SourceGraph CLI +http_file( + name = "scip-src", + executable = True, + sha256 = "eb3d627c4dad56b9f5d06634b90384104db065d410c4b338fa5e6558be4f3ad9", + urls = [ + # Implements retry by relisting each url multiple times to be used as a failover. + # TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries + "https://github.com/sourcegraph/src-cli/releases/download/6.4.0/src_linux_amd64", + ] * 5, +) + bazel_dep(name = "platforms", version = "0.0.9") single_version_override( module_name = "platforms", diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 92ed5addd2b..3beed525c8d 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"]) # Expose script for external usage through bazel. exports_files([ "install_rules.py", + "run_under_working_dir.sh", ]) sh_binary( diff --git a/bazel/run_under_working_dir.sh b/bazel/run_under_working_dir.sh new file mode 100755 index 00000000000..370b267bcf7 --- /dev/null +++ b/bazel/run_under_working_dir.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +RUNFILES_WORKING_DIRECTORY="$(pwd)" + +if [ -z $BUILD_WORKING_DIRECTORY ]; then + echo "ERROR: BUILD_WORKING_DIRECTORY was not set, was this run from bazel?" + exit 1 +fi + +cd $BUILD_WORKING_DIRECTORY + +${RUNFILES_WORKING_DIRECTORY}/${BINARY_PATH} "${@:1}" diff --git a/etc/evergreen_yml_components/tasks/compile_tasks.yml b/etc/evergreen_yml_components/tasks/compile_tasks.yml index 51fa2eb90d9..fccc62f21a1 100644 --- a/etc/evergreen_yml_components/tasks/compile_tasks.yml +++ b/etc/evergreen_yml_components/tasks/compile_tasks.yml @@ -179,6 +179,30 @@ tasks: --keep_going --build_tag_filters=${bazel_filters_for_cache_hydration} + - name: build_source_graph_index + tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - func: "bazel compile" + vars: + targets: compiledb + bazel_args: --intree_compdb + - func: "bazel compile" + vars: + targets: //src/... + bazel_args: --output_groups=compilation_outputs --build_enterprise=True --config=dbg --remote_download_outputs=all + - func: "bazel run" + vars: + target: //:scip-clang -- --compdb-path ./compile_commands.json + - func: "bazel run" + vars: + target: //:scip-src -- code-intel upload -file=index.scip + args: -github-token=${SOURCE_GRAPH_GITHUB_CLONER_TOKEN} + env: SRC_ENDPOINT=https://mongodb.sourcegraphcloud.com SRC_ACCESS_TOKEN=${SOURCE_GRAPH_API_ACCESS_TOKEN} + redact_args: true + - name: hydrate_bazel_profile_opt tags: [ @@ -974,6 +998,12 @@ task_groups: - hydrate_bazel_profile_dbg_aubsan - hydrate_bazel_profile_dbg_tsan + - <<: *compile_task_group_template + name: build_source_graph_index_TG + max_hosts: -1 + tasks: + - build_source_graph_index + - <<: *compile_bazel_task_group_template name: compile_bazel_windows_TG tasks: diff --git a/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_branch_only.yml index bddd865fab0..f68a6dc6a2c 100644 --- a/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/ubuntu/test_dev_master_branch_only.yml @@ -45,3 +45,17 @@ buildvariants: tasks: - name: hydrate_bazel_profile_TG - name: .clang_tidy + + - name: &upload_source_graph_index upload_source_graph_index + display_name: "~ Upload Source Graph Index" + tags: ["source_graph_indexing"] + cron: "30 * * * *" # Every hour, 30 min offset + # SourceGraph's indexer requires a lot of CPU to execute in a reasonable amount of time + run_on: + - rhel93-xxlarge + stepback: false + expansions: + compile_variant: *upload_source_graph_index + evergreen_remote_exec: on + tasks: + - name: build_source_graph_index_TG diff --git a/evergreen/bazel_run.sh b/evergreen/bazel_run.sh index 119eb8b70a8..87647391d6a 100644 --- a/evergreen/bazel_run.sh +++ b/evergreen/bazel_run.sh @@ -4,6 +4,8 @@ # Required environment variables: # * ${target} - Build target # * ${args} - Extra command line args to pass to "bazel run" +# * ${env} - Env variable string to set (ex. ENV_VAR_ABC=123) +# * ${redact_args} - If set, redact the args in the report # Needed for evergreen scripts that use evergreen expansions and utility methods. DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" @@ -17,7 +19,7 @@ set -o verbose activate_venv # Use `eval` to force evaluation of the environment variables in the echo statement: -eval echo "Execution environment: Args: ${args} Target: ${target}" +eval echo "Execution environment: Args: ${args} Target: ${target} Env: ${env} redact_args: ${redact_args}" source ./evergreen/bazel_utility_functions.sh source ./evergreen/bazel_RBE_supported.sh @@ -46,17 +48,23 @@ if [[ -f /etc/os-release ]]; then fi fi +if [[ -n "$redact_args" ]]; then + INVOCATION_WITH_REDACTION="${target}" +else + INVOCATION_WITH_REDACTION="${target} ${args}" +fi + # Print command being run to file that can be uploaded echo "python buildscripts/install_bazel.py" > bazel-invocation.txt -echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${args} ${target}" >> bazel-invocation.txt +echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >> bazel-invocation.txt # Run bazel command, retrying up to five times MAX_ATTEMPTS=5 for ((i = 1; i <= $MAX_ATTEMPTS; i++)); do - eval $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${args} ${target} >> bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 60 + eval $env $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${target} ${args} >> bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10 if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >> bazel_output.log 2>&1; fi $BAZEL_BINARY shutdown done -$python ./buildscripts/simple_report.py --test-name "bazel run ${args} ${target}" --log-file bazel_output.log --exit-code $RET +$python ./buildscripts/simple_report.py --test-name "bazel run ${INVOCATION_WITH_REDACTION}" --log-file bazel_output.log --exit-code $RET exit $RET