From 0b50acadb8c08bdce39ad9861d9fafd877a8ebac Mon Sep 17 00:00:00 2001 From: Philip Stoev Date: Tue, 7 Oct 2025 15:24:10 +0200 Subject: [PATCH] SERVER-111900 Update golden tests in bulk across all passthoughs (#42157) GitOrigin-RevId: b6d03948eb0e59189a0634f3d1ce93ae98c388d5 --- buildscripts/golden_test.py | 97 ++++++++++++++++---- buildscripts/resmokelib/configure_resmoke.py | 2 +- docs/golden_data_test_framework.md | 18 ++++ 3 files changed, 97 insertions(+), 20 deletions(-) diff --git a/buildscripts/golden_test.py b/buildscripts/golden_test.py index a79443271c8..111c0138bd7 100755 --- a/buildscripts/golden_test.py +++ b/buildscripts/golden_test.py @@ -13,7 +13,7 @@ import shutil import sys from dataclasses import dataclass from datetime import datetime -from subprocess import call, check_output +from subprocess import CalledProcessError, call, check_call, check_output import click @@ -313,6 +313,30 @@ class GoldenTestApp(object): "Skipping setting GOLDEN_TEST_CONFIG_PATH global variable, variable already defined." ) + def accept(self, output_name): + """Accept the actual test output and copy it as new golden data to the source repo.""" + + output = self.get_latest_or_matching_output(output_name) + self.vprint(f"Accepting actual results from output '{output.name}'") + + repo_root = self.get_git_root() + paths = self.get_paths(output.name) + + self.vprint(f"Copying files recursively from '{paths.actual}' to '{repo_root}'") + if not self.dry_run: + copytree_dirs_exist_ok(paths.actual, repo_root) + + def clean(self): + """Remove all test outputs.""" + + outputs = self.get_outputs() + self.vprint(f"Deleting {len(outputs)} outputs") + for output in outputs: + output_path = self.get_output_path(output.name) + self.vprint(f"Deleting folder: '{output_path}'") + if not self.dry_run: + shutil.rmtree(output_path) + @cli.command("diff", help="Diff the expected and actual folders of the test output") @click.argument("output_name", required=False) @click.pass_obj @@ -349,30 +373,14 @@ class GoldenTestApp(object): def command_accept(self, output_name): """Accept the actual test output and copy it as new golden data to the source repo.""" self.init_config() - - output = self.get_latest_or_matching_output(output_name) - self.vprint(f"Accepting actual results from output '{output.name}'") - - repo_root = self.get_git_root() - paths = self.get_paths(output.name) - - self.vprint(f"Copying files recursively from '{paths.actual}' to '{repo_root}'") - if not self.dry_run: - copytree_dirs_exist_ok(paths.actual, repo_root) + self.accept(output_name) @cli.command("clean", help="Remove all test outputs") @click.pass_obj def command_clean(self): """Remove all test outputs.""" self.init_config() - - outputs = self.get_outputs() - self.vprint(f"Deleting {len(outputs)} outputs") - for output in outputs: - output_path = self.get_output_path(output.name) - self.vprint(f"Deleting folder: '{output_path}'") - if not self.dry_run: - shutil.rmtree(output_path) + self.clean() @cli.command("latest", help="Get the name of the most recent test output") @click.pass_obj @@ -403,6 +411,57 @@ class GoldenTestApp(object): else: raise AppError(f"Platform not supported by this setup utility: {platform.platform()}") + @cli.command("clean-run-accept", help="Runs the test in all suites and accepts the results.") + @click.argument("test_name", required=True) + @click.pass_obj + def command_clean_run_accept(self, test_name): + """Runs a given jstest with all its passthrough suites and accepts the results.""" + self.init_config() + self.clean() + + self.vprint( + f"Obtaining the list of suites {test_name} belongs to using 'resmoke.py find-suites' ..." + ) + suites = ( + check_output(["buildscripts/resmoke.py", "find-suites", test_name], text=True) + .strip() + .split() + ) + assert len(suites) > 0, f"Failed to find any suites for test {test_name}" + self.vprint(f"Found suites {suites} for test {test_name}") + + resmoke_invocations = [] + + for suite in suites: + resmoke_invocations.append(["--suite", suite]) + + if len(suites) == 1 and suites[0] == "query_golden_classic": + # The test only belongs to the query_golden_classic passthrough, which means that its various expected files + # are generated by different evergreen build variants, not by different passthroughs. We could try to extract + # the correct resmoke arguments from the evergreen .yml file, but in practice there are many passthroughs and + # most define resmoke arguments that have nothing to do with query golden testing. So we hardcore the list + # of resmoke arguments here. + self.vprint( + "Only query_golden_classic passthrough found, will run with various settings for internalQueryFrameworkControl" + ) + for framework_control in [ + ["--runAllFeatureFlagTests", "--excludeWithAnyTags=featureFlagSbeFull"], + ["--mongodSetParameters={internalQueryFrameworkControl: forceClassicEngine}"], + ["--mongodSetParameters={internalQueryFrameworkControl: trySbeEngine}"], + ["--mongodSetParameters={internalQueryFrameworkControl: trySbeRestricted}"], + ]: + resmoke_invocations.append(["--suite", suites[0], *framework_control]) + + for resmoke_invocation in resmoke_invocations: + self.vprint(f"Will run resmoke.py with arguments: {resmoke_invocation}") + + for resmoke_invocation in resmoke_invocations: + try: + check_call(["buildscripts/resmoke.py", "run", *resmoke_invocation, test_name]) + except CalledProcessError: + # Golden test failed, accept the new results + self.accept(None) + def main(): """Execute main.""" diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index 10f87416503..e79bd0e260d 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -850,7 +850,7 @@ flags in common: {common_set} # If there is some problem setting up metrics we don't want resmoke to fail # We would rather just swallow the error traceback.print_exc() - print("Failed to set up otel metrics. Continuing.") + print("Failed to set up otel metrics. Continuing.", file=sys.stderr) # Force invalid suite config _config.FORCE_EXCLUDED_TESTS = config.pop("force_excluded_tests") diff --git a/docs/golden_data_test_framework.md b/docs/golden_data_test_framework.md index f497e7ee67e..f15e1ee7e01 100644 --- a/docs/golden_data_test_framework.md +++ b/docs/golden_data_test_framework.md @@ -293,6 +293,24 @@ Get all available commands and options: $> buildscripts/golden_test.py --help ``` +### Update multiple expected files at once + +Some tests will run in multiple passthroughs or build variants, so they have multiple expected files. + +Whenever the test is updated, all the expected files should be updated together as well. + +```bash +buildscripts/golden_test.py --verbose clean-run-accept jstests/query_golden/NAME_OF_TEST.js +``` + +This option uses `resmoke.py find-suites` to determine the passthrough suites a test belongs to and +runs them. + +If the test is found to only belong to the `query_golden_classic` passthrough, it is assumed that +it can have multiple expected results due to being run under multiple build variants with a different +`internalQueryFrameworkControl` settings. So the test will be run with various values for +`internalQueryFrameworkControl`. + # How to diff test results from a non-workstation test run ## Bulk folder diff the results: