SERVER-103719 Add ability to run Coverity scans on Evergreen (#34920)

Co-authored-by: Daniel Moody <dmoody256@gmail.com>
GitOrigin-RevId: fe1df1fc5b7e881de30567acdb51ee60e42ab471

bazel/coverity/coverity_toolchain.bzl

@@ -1,12 +1,15 @@
 def _coverity_toolchain(ctx):
-    result = ctx.execute([
+    retCode = 1
-        "ls",
+    if "COVERITY_INSTALL_ROOT" in ctx.os.environ:
-        "/data/cov-sa/bin/cov-build",
+        result = ctx.execute([
-    ])
+            "ls",
+            ctx.getenv("COVERITY_INSTALL_ROOT") + "/bin/cov-build",
+        ])
+        retCode = result.return_code
 
-    if result.return_code == 0:
+    if retCode == 0:
         ctx.report_progress("extracting coverity rules...")
-        result = ctx.download_and_extract("file:///data/cov-sa/bazel/rules_coverity.tar.gz")
+        result = ctx.download_and_extract("file://" + ctx.getenv("COVERITY_INSTALL_ROOT") + "/bazel/rules_coverity.tar.gz")
     else:
         ctx.template(
             "coverity/BUILD.bazel",

etc/coverity.yml

@@ -0,0 +1,57 @@
+# Coverity analysis options that are passed to cov-analyze command line
+analyze:
+  checkers:
+    all: true # Enables all checkers except a few which need to be explicitly enabled. See: https://documentation.blackduck.com/bundle/coverity-docs/page/commands/topics/cov-analyze.html#ariaid-title11
+    rule: true # Enables "rule" checkers. See: https://documentation.blackduck.com/bundle/coverity-docs/page/commands/topics/cov-analyze.html#refsection_dgq_pf2_4qb__list-checkers
+    checker-config:
+      # Explicitly disabled checkers
+      # Add any checker that needs to be disabled below
+      STACK_USE:
+        enabled: false
+      COPY_INSTEAD_OF_MOVE:
+        enabled: false
+      RULE_OF_ZERO_THREE_FIVE:
+        enabled: false
+      MISSING_RETURN:
+        enabled: false
+      SLEEP:
+        enabled: false
+      PASS_BY_VALUE:
+        enabled: false
+      GLOBAL_INIT_ORDER:
+        enabled: false
+      # Explicitly enabled checkers
+      # Add any checker that needs to be explicitly enabled below
+      REVERSE_OVERRUN:
+        enabled: true
+      # Checkers with customized options
+      CHECKED_RETURN:
+        enabled: true
+        options:
+          stat_threshold: 80
+      DEADCODE:
+        enabled: true
+        options:
+          no_dead_default: true
+          report_redundant_tests: true
+      NULL_RETURNS:
+        enabled: true
+        options:
+          stat_threshold: 50
+  # Enables additional filtering of potential defects by using an additional false-path pruner (FPP). More details here https://documentation.blackduck.com/bundle/coverity-docs/page/commands/topics/cov-analyze.html#ariaid-title5
+  constraint-fpp: true
+  # Enables function pointer models if the analysis fails to analyze certain function pointers calls. More details at https://documentation.blackduck.com/bundle/coverity-docs/page/commands/topics/cov-analyze.html#ariaid-title9
+  c-cpp-fnptr: true
+  # Enables full, virtual-call resolution. More details at https://documentation.blackduck.com/bundle/coverity-docs/page/commands/topics/cov-analyze.html#ariaid-title9
+  c-cpp-virtual: true
+  # Enables call graph matrics. The callgraph-metrics files store information about which functions are analyzed. More details at https://documentation.blackduck.com/bundle/coverity-docs/page/commands/topics/cov-analyze.html#ariaid-title20
+  callgraph-metrics: true
+  cov-analyze-args:
+    # Specifies the name for the configuration file, which allows you to change the parse warnings that pass through a warning filter. More details at https://documentation.blackduck.com/bundle/coverity-docs/page/commands/topics/cov-analyze.html#ariaid-title9
+    - --parse-warnings-config
+    - etc/coverity_parse_warnings.conf
+
+commit:
+  connect:
+    stream: mongo.master
+    project: "MongoDB master"

etc/evergreen_nightly.yml

@@ -51,6 +51,9 @@ include:
   ### Uncomment when using this file for a LTS release branch. ###
   # - filename: etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml
 
+  - filename: etc/evergreen_yml_components/tasks/coverity_tasks.yml
+  - filename: etc/evergreen_yml_components/variants/coverity.yml
+
 parameters:
   - key: evergreen_config_file_path
     value: "etc/evergreen_nightly.yml"

etc/evergreen_yml_components/tasks/coverity_tasks.yml

@@ -0,0 +1,94 @@
+functions:
+  f_clone_source:
+    - &clone_source
+      display_name: "Cloning source to analyze"
+      command: git.get_project
+      params:
+        directory: src
+        clone_depth: 1
+        recurse_submodules: true
+
+    - &check
+      display_name: "Checking if project has needed Coverity config"
+      command: subprocess.exec
+      params:
+        add_expansions_to_env: true
+        redirect_standard_error_to_output: true
+        binary: "${module_prefix}/${module_name}/evergreen/coverity.sh"
+        args:
+          - "check"
+
+  f_download_and_extract_coverity:
+    - &assume_role
+      display_name: "Assuming IAM role"
+      command: ec2.assume_role
+      params:
+        role_arn: arn:aws:iam::557821124784:role/evergreen.evergreen.coverity
+
+    - &download_coverity_license
+      display_name: "downloading Coverity license file"
+      command: s3.get
+      params:
+        aws_key: ${AWS_ACCESS_KEY_ID}
+        aws_secret: ${AWS_SECRET_ACCESS_KEY}
+        aws_session_token: ${AWS_SESSION_TOKEN}
+        remote_file: license.dat
+        bucket: evergreen.coverity
+        local_file: license.dat
+
+    - &download_coverity_binary
+      display_name: "downloading Coverity binary"
+      command: s3.get
+      params:
+        aws_key: ${AWS_ACCESS_KEY_ID}
+        aws_secret: ${AWS_SECRET_ACCESS_KEY}
+        aws_session_token: ${AWS_SESSION_TOKEN}
+        remote_file: linux/x86_64/latest/coverity.tar.gz
+        bucket: evergreen.coverity
+        local_file: coverity.tar.gz
+
+    - &extract_coverity_binary
+      display_name: "Extracting Coverity binary tarball"
+      command: subprocess.exec
+      params:
+        add_expansions_to_env: true
+        redirect_standard_error_to_output: true
+        binary: "${module_prefix}/${module_name}/evergreen/coverity.sh"
+        args:
+          - "extract"
+
+  # Analyzes after build
+  f_analyze:
+    - command: subprocess.exec
+      params:
+        add_expansions_to_env: true
+        redirect_standard_error_to_output: true
+        binary: "${module_prefix}/${module_name}/evergreen/coverity.sh"
+        args:
+          - "analyze"
+
+  # Pushes results to Coverity server after a successful analysis
+  f_commit:
+    - command: subprocess.exec
+      params:
+        add_expansions_to_env: true
+        redirect_standard_error_to_output: true
+        binary: "${module_prefix}/${module_name}/evergreen/coverity.sh"
+        args:
+          - "commit"
+
+  # All in one function, from cloning to pushing results to Coverity server
+  f_run_coverity:
+    - *clone_source
+    - *check
+    - *assume_role
+    - *download_coverity_license
+    - *download_coverity_binary
+    - *extract_coverity_binary
+    - command: subprocess.exec
+      params:
+        add_expansions_to_env: true
+        redirect_standard_error_to_output: true
+        binary: "${module_prefix}/${module_name}/evergreen/coverity.sh"
+        args:
+          - "scan"

etc/evergreen_yml_components/variants/coverity.yml

@@ -0,0 +1,90 @@
+buildvariants:
+  - name: bv_coverity_analysis
+    display_name: Coverity Analysis
+    # Don't run Coverity analysis as part of patch builds
+    patchable: false
+    allow_for_git_tag: false
+    # Run at 4 am UTC everyday
+    cron: "0 4 * * *"
+    run_on: ubuntu2404-xlarge
+    modules:
+      - devprod_coverity
+    tasks:
+      - name: run_coverity
+
+functions:
+  f_generate_evergreen_bazelrc:
+    command: subprocess.exec
+    display_name: "generate evergreen bazelrc"
+    params:
+      binary: bash
+      args:
+        - "src/evergreen/generate_evergreen_bazelrc.sh"
+
+  f_setup_python:
+    command: subprocess.exec
+    display_name: "set up venv"
+    params:
+      binary: bash
+      args:
+        - "src/evergreen/functions/venv_setup.sh"
+
+  f_coverity_build:
+    command: subprocess.exec
+    display_name: "Coverity build"
+    params:
+      binary: bash
+      args:
+        - "src/evergreen/coverity_build.sh"
+      env:
+        COVERITY_INSTALL_ROOT: ${workdir}/coverity
+      add_expansions_to_env: true
+
+tasks:
+  - name: run_coverity
+    tags:
+      [
+        "assigned_to_jira_team_devprod_build",
+        "development_critical_single_variant",
+        "requires_large_host",
+        "coverity",
+      ]
+    exec_timeout_secs: 28800 # Max scan time of 8 hours
+    commands:
+      - func: f_clone_source
+        vars:
+          module_prefix: ${workdir}/devprodCoveritySrc
+          module_name: devprod_coverity
+      # Functions to prepare for Coverity build
+      - func: f_expansions_write
+      - func: f_setup_python
+      - func: f_expansions_write
+      - func: f_generate_evergreen_bazelrc
+      - func: f_download_and_extract_coverity
+        vars:
+          module_prefix: ${workdir}/devprodCoveritySrc
+          module_name: devprod_coverity
+      - func: f_coverity_build
+        timeout_secs: 10800 # Idle timeout of 3 hours
+      - func: f_analyze
+        vars:
+          module_prefix: ${workdir}/devprodCoveritySrc
+          module_name: devprod_coverity
+        timeout_secs: 10800 # Idle timeout of 3 hours
+      - func: f_commit
+        vars:
+          module_prefix: ${workdir}/devprodCoveritySrc
+          module_name: devprod_coverity
+
+#######################################
+#               Modules               #
+#######################################
+# for some reason this has can't be included from the included configuration file above
+# it may be because one of the includes needs this module, could be an evergreen bug
+modules:
+  - name: devprod_coverity
+    owner: mongodb-devprod-infrastructure
+    repo: coverity
+    prefix: "${workdir}/devprodCoveritySrc"
+    branch: main
+    auto_update: true

evergreen/coverity_build.sh

@@ -0,0 +1,34 @@
+#!/bin/env bash
+
+set -eo pipefail
+
+cd src
+
+. evergreen/prelude_venv.sh
+activate_venv
+python buildscripts/install_bazel.py
+bazel_bin="$HOME/.local/bin/bazelisk"
+# number of parallel jobs to use for build.
+# Even with scale=0 (the default), bc command adds decimal digits in case of multiplication. Division by 1 gives us a whole number with scale=0
+bazel_jobs=$(bc <<< "$(grep -c '^processor' /proc/cpuinfo) * .85 / 1")
+build_config="--config=local --jobs=$bazel_jobs --compiler_type=gcc --opt=off --dbg=False --allocator=system"
+bazel_query='mnemonic("CppCompile|LinkCompile", filter(//src/mongo, deps(//:install-core)) except //src/mongo/db/modules/enterprise/src/streams/third_party/...)'
+bazel_cache="--output_user_root=$workdir/bazel_cache"
+
+python bazel/coverity/generate_coverity_command.py --bazel_executable=$bazel_bin --bazel_cache=$bazel_cache --bazel_query="$bazel_query" $build_config --noinclude_artifacts
+$bazel_bin $bazel_cache build $build_config --build_tag_filters=gen_source //src/...
+bazelBuildCommand="$bazel_bin $bazel_cache build $build_config //src/mongo/db/modules/enterprise/coverity:enterprise_coverity_build"
+echo "Bazel Build Command: $bazelBuildCommand"
+covIdir="$workdir/covIdir"
+if [ -d "$covIdir" ]; then
+  echo "covIdir already exists, meaning idir extracted after download from S3"
+else
+  mkdir $workdir/covIdir
+fi
+$workdir/coverity/bin/cov-build --dir "$covIdir" --verbose 0 -j $bazel_jobs --return-emit-failures --parse-error-threshold=99 --bazel $bazelBuildCommand
+ret=$?
+if [ $ret -ne 0 ]; then
+  echo "cov-build faild with exit code $ret"
+else
+  echo "cov-build was successful"
+fi