SERVER-111900 Update golden tests in bulk across all passthoughs (#42157)

GitOrigin-RevId: b6d03948eb0e59189a0634f3d1ce93ae98c388d5

buildscripts/golden_test.py

@@ -13,7 +13,7 @@ import shutil
 import sys
 from dataclasses import dataclass
 from datetime import datetime
-from subprocess import call, check_output
+from subprocess import CalledProcessError, call, check_call, check_output
 
 import click
 
@@ -313,6 +313,30 @@ class GoldenTestApp(object):
                 "Skipping setting GOLDEN_TEST_CONFIG_PATH global variable, variable already defined."
             )
 
+    def accept(self, output_name):
+        """Accept the actual test output and copy it as new golden data to the source repo."""
+
+        output = self.get_latest_or_matching_output(output_name)
+        self.vprint(f"Accepting actual results from output '{output.name}'")
+
+        repo_root = self.get_git_root()
+        paths = self.get_paths(output.name)
+
+        self.vprint(f"Copying files recursively from '{paths.actual}' to '{repo_root}'")
+        if not self.dry_run:
+            copytree_dirs_exist_ok(paths.actual, repo_root)
+
+    def clean(self):
+        """Remove all test outputs."""
+
+        outputs = self.get_outputs()
+        self.vprint(f"Deleting {len(outputs)} outputs")
+        for output in outputs:
+            output_path = self.get_output_path(output.name)
+            self.vprint(f"Deleting folder: '{output_path}'")
+            if not self.dry_run:
+                shutil.rmtree(output_path)
+
     @cli.command("diff", help="Diff the expected and actual folders of the test output")
     @click.argument("output_name", required=False)
     @click.pass_obj
@@ -349,30 +373,14 @@ class GoldenTestApp(object):
     def command_accept(self, output_name):
         """Accept the actual test output and copy it as new golden data to the source repo."""
         self.init_config()
-
+        self.accept(output_name)
-        output = self.get_latest_or_matching_output(output_name)
-        self.vprint(f"Accepting actual results from output '{output.name}'")
-
-        repo_root = self.get_git_root()
-        paths = self.get_paths(output.name)
-
-        self.vprint(f"Copying files recursively from '{paths.actual}' to '{repo_root}'")
-        if not self.dry_run:
-            copytree_dirs_exist_ok(paths.actual, repo_root)
 
     @cli.command("clean", help="Remove all test outputs")
     @click.pass_obj
     def command_clean(self):
         """Remove all test outputs."""
         self.init_config()
-
+        self.clean()
-        outputs = self.get_outputs()
-        self.vprint(f"Deleting {len(outputs)} outputs")
-        for output in outputs:
-            output_path = self.get_output_path(output.name)
-            self.vprint(f"Deleting folder: '{output_path}'")
-            if not self.dry_run:
-                shutil.rmtree(output_path)
 
     @cli.command("latest", help="Get the name of the most recent test output")
     @click.pass_obj
@@ -403,6 +411,57 @@ class GoldenTestApp(object):
         else:
             raise AppError(f"Platform not supported by this setup utility: {platform.platform()}")
 
+    @cli.command("clean-run-accept", help="Runs the test in all suites and accepts the results.")
+    @click.argument("test_name", required=True)
+    @click.pass_obj
+    def command_clean_run_accept(self, test_name):
+        """Runs a given jstest with all its passthrough suites and accepts the results."""
+        self.init_config()
+        self.clean()
+
+        self.vprint(
+            f"Obtaining the list of suites {test_name} belongs to using 'resmoke.py find-suites' ..."
+        )
+        suites = (
+            check_output(["buildscripts/resmoke.py", "find-suites", test_name], text=True)
+            .strip()
+            .split()
+        )
+        assert len(suites) > 0, f"Failed to find any suites for test {test_name}"
+        self.vprint(f"Found suites {suites} for test {test_name}")
+
+        resmoke_invocations = []
+
+        for suite in suites:
+            resmoke_invocations.append(["--suite", suite])
+
+        if len(suites) == 1 and suites[0] == "query_golden_classic":
+            # The test only belongs to the query_golden_classic passthrough, which means that its various expected files
+            # are generated by different evergreen build variants, not by different passthroughs. We could try to extract
+            # the correct resmoke arguments from the evergreen .yml file, but in practice there are many passthroughs and
+            # most define resmoke arguments that have nothing to do with query golden testing. So we hardcore the list
+            # of resmoke arguments here.
+            self.vprint(
+                "Only query_golden_classic passthrough found, will run with various settings for internalQueryFrameworkControl"
+            )
+            for framework_control in [
+                ["--runAllFeatureFlagTests", "--excludeWithAnyTags=featureFlagSbeFull"],
+                ["--mongodSetParameters={internalQueryFrameworkControl: forceClassicEngine}"],
+                ["--mongodSetParameters={internalQueryFrameworkControl: trySbeEngine}"],
+                ["--mongodSetParameters={internalQueryFrameworkControl: trySbeRestricted}"],
+            ]:
+                resmoke_invocations.append(["--suite", suites[0], *framework_control])
+
+        for resmoke_invocation in resmoke_invocations:
+            self.vprint(f"Will run resmoke.py with arguments: {resmoke_invocation}")
+
+        for resmoke_invocation in resmoke_invocations:
+            try:
+                check_call(["buildscripts/resmoke.py", "run", *resmoke_invocation, test_name])
+            except CalledProcessError:
+                # Golden test failed, accept the new results
+                self.accept(None)
+
 
 def main():
     """Execute main."""

buildscripts/resmokelib/configure_resmoke.py

@@ -850,7 +850,7 @@ flags in common: {common_set}
             # If there is some problem setting up metrics we don't want resmoke to fail
             # We would rather just swallow the error
             traceback.print_exc()
-            print("Failed to set up otel metrics. Continuing.")
+            print("Failed to set up otel metrics. Continuing.", file=sys.stderr)
 
     # Force invalid suite config
     _config.FORCE_EXCLUDED_TESTS = config.pop("force_excluded_tests")

docs/golden_data_test_framework.md

@@ -293,6 +293,24 @@ Get all available commands and options:
 $> buildscripts/golden_test.py --help
 ```
 
+### Update multiple expected files at once
+
+Some tests will run in multiple passthroughs or build variants, so they have multiple expected files.
+
+Whenever the test is updated, all the expected files should be updated together as well.
+
+```bash
+buildscripts/golden_test.py --verbose clean-run-accept jstests/query_golden/NAME_OF_TEST.js
+```
+
+This option uses `resmoke.py find-suites` to determine the passthrough suites a test belongs to and
+runs them.
+
+If the test is found to only belong to the `query_golden_classic` passthrough, it is assumed that
+it can have multiple expected results due to being run under multiple build variants with a different
+`internalQueryFrameworkControl` settings. So the test will be run with various values for
+`internalQueryFrameworkControl`.
+
 # How to diff test results from a non-workstation test run
 
 ## Bulk folder diff the results: