diff --git a/.gitignore b/.gitignore index 1175fd29129..cd5e71d0d10 100644 --- a/.gitignore +++ b/.gitignore @@ -280,10 +280,6 @@ package-lock.json # jstestfuzz generated test directory /jstestfuzz/ -# docker volumes mapped in lint_fuzzer_sanity_patch.py -/jstestfuzzinput -/jstestfuzzoutput - # Bazel-related work # (Note that each string does NOT terminate with a '/' so that gitignore works properly) bazel-* diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 6e6005a516e..9e7694d9e5e 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -1977,9 +1977,14 @@ functions: "setup jstestfuzz": - *f_expansions_write - - command: ec2.assume_role + - command: github.generate_token params: - role_arn: "arn:aws:iam::901841024863:role/ecr-role-evergreen-ro" + owner: 10gen + repo: jstestfuzz + expansion_name: github_token + permissions: + metadata: read + contents: read - command: subprocess.exec display_name: "jstestfuzz setup sh" params: @@ -1987,13 +1992,7 @@ functions: args: - "./src/evergreen/jstestfuzz_setup.sh" include_expansions_in_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - command: expansions.update - params: - file: src/expansions.yml - - *f_expansions_write + - github_token "setup query_tester test repos": - *f_expansions_write @@ -2033,8 +2032,6 @@ functions: binary: bash args: - "src/evergreen/lint_fuzzer_sanity_patch.sh" - include_expansions_in_env: - - CONTAINER_RUNTIME "lint fuzzer sanity all": - *f_expansions_write @@ -2044,8 +2041,6 @@ functions: binary: bash args: - "./src/evergreen/lint_fuzzer_sanity_all.sh" - include_expansions_in_env: - - CONTAINER_RUNTIME "monitor mongo fork 10gen": - *f_expansions_write @@ -2095,8 +2090,6 @@ functions: binary: bash args: - "./src/evergreen/jstestfuzz_run.sh" - include_expansions_in_env: - - CONTAINER_RUNTIME - command: archive.targz_pack display_name: "archive jstests.tgz" params: diff --git a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml index 1ceda305f76..ee7b4391795 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/non_server_teams/tasks.yml @@ -367,7 +367,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 8 num_tasks: 2 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: //buildscripts/resmokeconfig:jstestfuzz npm_command: jstestfuzz bazel_args: >- diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml index 2e57cd3cb9f..11a6724035d 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml @@ -2513,7 +2513,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded_causal_consistency resmoke_args: --numClientsPerFixture=10 @@ -2538,7 +2538,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 2 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded_kill_terminate_stepdown resmoke_args: --numClientsPerFixture=10 @@ -2567,7 +2567,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 2 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded_kill_terminate_stepdown resmoke_args: >- --numClientsPerFixture=10 @@ -2651,7 +2651,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded resmoke_args: --numClientsPerFixture=10 @@ -2674,7 +2674,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded_notablescan resmoke_args: --numClientsPerFixture=10 @@ -2696,7 +2696,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded_causal_consistency resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -2721,7 +2721,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded_kill_terminate_stepdown resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -2745,7 +2745,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -2769,7 +2769,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests resmoke_args: >- --mongodSetParameters='{logComponentVerbosity: {command: 2}}' --runNoFeatureFlagTests @@ -2792,7 +2792,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded_notablescan resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -2814,7 +2814,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded_with_config_transitions_and_add_remove_shard resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml index 55150a619f0..69c4c599626 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/durable_transactions_and_availability/tasks.yml @@ -721,7 +721,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_replication resmoke_args: --numClientsPerFixture=10 @@ -744,7 +744,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_replication_notablescan resmoke_args: --numClientsPerFixture=10 @@ -766,7 +766,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_replication_write_conflicts resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}' --numClientsPerFixture=10" @@ -791,7 +791,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 8 num_tasks: 5 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_replication_fcbis resmoke_args: --storageEngine=wiredTiger name: jstestfuzz_replication_fcbis @@ -818,7 +818,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_replication resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -842,7 +842,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 8 num_tasks: 5 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_replication_initsync resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -870,7 +870,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests resmoke_args: >- --mongodSetParameters='{logComponentVerbosity: {command: 2}}' --runNoFeatureFlagTests @@ -893,7 +893,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_replication_notablescan resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -915,7 +915,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_replication_write_conflicts resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -1843,7 +1843,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_replication resmoke_args: >- --numClientsPerFixture=10 @@ -2056,7 +2056,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: ${jstestfuzz_concurrent_num_files|10} num_tasks: 5 - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz resmoke_args: --numClientsPerFixture=10 @@ -2078,7 +2078,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_interrupt resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -2100,7 +2100,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_interrupt_replication resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml index 4a2231d1912..b54964d195d 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml @@ -435,7 +435,7 @@ tasks: suite: generational_fuzzer use_large_distro: "true" resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" - jstestfuzz_vars: --diffTestingMode timeseries --numPipelines=1000 --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0088cfcbef520f248803e17284677 + jstestfuzz_vars: --diffTestingMode timeseries --numPipelines=1000 --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0 ## Standalone fuzzer for checking block_processing correctness ## - <<: *jstestfuzz_template @@ -1157,7 +1157,7 @@ tasks: - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars - jstestfuzz_vars: --jsTestsDir jstests + jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" npm_command: jstestfuzz @@ -1638,7 +1638,7 @@ tasks: resmoke_args: >- --mongodSetParameters='{logComponentVerbosity: {command: 2}}' --runNoFeatureFlagTests - jstestfuzz_vars: --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0088cfcbef520f248803e17284677 + jstestfuzz_vars: --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0 # jstestfuzz standalone update time-series generational fuzzer ## - <<: *jstestfuzz_template @@ -1890,7 +1890,7 @@ tasks: suite: generational_fuzzer use_large_distro: "true" resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" - jstestfuzz_vars: --diffTestingMode optimization --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0088cfcbef520f248803e17284677 + jstestfuzz_vars: --diffTestingMode optimization --metaSeed 1726779665485 --jstestfuzzGitRev 8ddd8f12cab0 - <<: *task_template name: aggregation_repeat_queries_multiplan_single_solutions diff --git a/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml b/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml index 1193889556f..9430739c09f 100644 --- a/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml +++ b/etc/evergreen_yml_components/variants/windows/test_dev_master_and_lts_branches_only.yml @@ -46,8 +46,7 @@ buildvariants: - name: .jscore .common !.decimal !.sharding .requires_large_host distros: - windows-2022-large - # SERVER-105328: Enable this task once container support is added to Windows variants. - # - name: .jstestfuzz .common !.feature_flag_guarded + - name: .jstestfuzz .common !.feature_flag_guarded - name: .read_write_concern .linearize !.durable_history - name: replica_sets_auth_gen - name: replica_sets_jscore_passthrough_gen diff --git a/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml b/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml index 39e6c05cd80..8e749b9af91 100644 --- a/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml +++ b/etc/evergreen_yml_components/variants/wiredtiger/test_dev_master_branch_only.yml @@ -172,8 +172,7 @@ buildvariants: - name: .jscore .common !.sharding .requires_large_host distros: - windows-2022-large - # SERVER-105328: Enable this task once container support is added to Windows variants. - # - name: .jstestfuzz .common !.feature_flag_guarded + - name: .jstestfuzz .common !.feature_flag_guarded - name: replica_sets_auth_gen - name: sasl - name: sharding_auth_audit_gen diff --git a/evergreen/jstestfuzz_run.sh b/evergreen/jstestfuzz_run.sh index b810ec5e7fa..03f971a64b7 100644 --- a/evergreen/jstestfuzz_run.sh +++ b/evergreen/jstestfuzz_run.sh @@ -3,16 +3,29 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" set -evo pipefail +cd src/jstestfuzz + # Store 'jstestfuzz_vars' into 'vars'. We will use 'vars' instead of 'jstestfuzz_vars' for # the rest of this shell script. vars="${jstestfuzz_vars}" -# # If the "--jstestfuzzGitRev" option is present in 'vars', copy the option's value into -# # 'jstestfuzz_gitrev' and remove it from 'vars'. +# If the "--jstestfuzzGitRev" option is present in 'vars', copy the option's value into +# 'jstestfuzz_gitrev' and remove it from 'vars', and then reset the jstestfuzz repo to the +# specified git revision. echo "${vars}" | grep -q -- '--jstestfuzzGitRev[ \t]\+[A-Za-z0-9]\+' && RET=0 || RET=$? if [ $RET -eq 0 ]; then jstestfuzz_gitrev=$(echo "${vars}" | sed -e 's/.*--jstestfuzzGitRev[ \t]\+\([A-Za-z0-9]\+\).*/\1/') vars=$(echo "${vars}" | sed -e 's/\(.*\)--jstestfuzzGitRev[ \t]\+[A-Za-z0-9]\+\(.*\)/\1\2/') + + for i in {1..5}; do + git reset --hard "${jstestfuzz_gitrev}" && RET=0 && break || RET=$? && sleep 5 + echo "Failed to reset jstestfuzz to git revision ${jstestfuzz_gitrev}, retrying..." + done + + if [ $RET -ne 0 ]; then + echo "Failed to reset jstestfuzz to git revision ${jstestfuzz_gitrev}" + exit $RET + fi fi # If the "--metaSeed" option is present in 'vars', copy the option's value into 'meta_seed' @@ -60,10 +73,4 @@ if [[ "${npm_command}" != "jstestfuzz" ]]; then maybe_use_es_modules="--useEsModules" fi -mkdir -p ./src/jstestfuzz/out - -$CONTAINER_RUNTIME run --rm \ - -v "./src/jstestfuzz/out:/app/out" \ - -v "./src/jstests:/app/jstests" \ - 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:${jstestfuzz_gitrev:-latest} \ - npm run-script ${npm_command} -- ${vars} ${generated_seed_flag} ${in_patch_build_flag} ${maybe_use_es_modules} --branch ${branch_name} +./src/scripts/npm_run.sh ${npm_command} -- ${vars} ${generated_seed_flag} ${in_patch_build_flag} ${maybe_use_es_modules} --branch ${branch_name} diff --git a/evergreen/jstestfuzz_setup.sh b/evergreen/jstestfuzz_setup.sh index 88766487ce7..d05284f3566 100644 --- a/evergreen/jstestfuzz_setup.sh +++ b/evergreen/jstestfuzz_setup.sh @@ -6,18 +6,12 @@ cd src set -o errexit set -o verbose -if command -v docker >/dev/null 2>&1; then - echo "Docker is installed, using docker" - CONTAINER_RUNTIME=docker +for i in {1..5}; do + git clone https://x-access-token:${github_token}@github.com/10gen/jstestfuzz.git && RET=0 && break || RET=$? && sleep 5 + echo "Failed to clone github.com:10gen/jstestfuzz.git, retrying..." +done -elif command -v podman >/dev/null 2>&1; then - echo "Podman is installed, using podman" - CONTAINER_RUNTIME=podman -else - echo "Neither Docker nor Podman is installed. Please install one of them." - exit 1 +if [ $RET -ne 0 ]; then + echo "Failed to clone git@github.com:10gen/jstestfuzz.git" + exit $RET fi - -echo "CONTAINER_RUNTIME: ${CONTAINER_RUNTIME}" >>expansions.yml - -aws ecr get-login-password --region us-east-1 | $CONTAINER_RUNTIME login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com diff --git a/evergreen/lint_fuzzer_sanity_all.sh b/evergreen/lint_fuzzer_sanity_all.sh index 1608c1611ec..9ee902413ed 100644 --- a/evergreen/lint_fuzzer_sanity_all.sh +++ b/evergreen/lint_fuzzer_sanity_all.sh @@ -1,23 +1,22 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" -cd src +cd src/jstestfuzz set -o pipefail set -o verbose -LOCAL_INPUT_DIR="$PWD" -CONTAINER_INPUT_DIR="/app" - -# Pull first which will do the install then allow parallel processing below -podman pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest +# Run first with help which will do the install +# Then we can run it in parallel +./src/scripts/npm_run.sh --help # Run parse-jsfiles on 50 files at a time with 32 processes in parallel. # Skip javascript files in third_party directory -find "jstests" "src/mongo/db/modules/enterprise" -path "$PWD/jstests/third_party" -prune -o -name "*.js" -print | - xargs -P 32 -L 50 $CONTAINER_RUNTIME run -v $LOCAL_INPUT_DIR/jstests:$CONTAINER_INPUT_DIR/jstests -v $LOCAL_INPUT_DIR/src/mongo/db/modules/enterprise:$CONTAINER_INPUT_DIR/src/mongo/db/modules/enterprise 901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest npm run-script parse-jsfiles -- 2>&1 | - tee lint_fuzzer_sanity.log +find "$PWD/../jstests" "$PWD/../src/mongo/db/modules/enterprise" -path "$PWD/../jstests/third_party" -prune -o -name "*.js" -print | xargs -P 32 -L 50 ./src/scripts/npm_run.sh parse-jsfiles -- 2>&1 | tee lint_fuzzer_sanity.log exit_code=$? +# Exit out of the jstestfuzz directory +cd .. + activate_venv -$python ./buildscripts/simple_report.py --test-name lint_fuzzer_sanity_all --log-file lint_fuzzer_sanity.log --exit-code $exit_code +$python ./buildscripts/simple_report.py --test-name lint_fuzzer_sanity_all --log-file jstestfuzz/lint_fuzzer_sanity.log --exit-code $exit_code exit $exit_code diff --git a/evergreen/lint_fuzzer_sanity_patch.py b/evergreen/lint_fuzzer_sanity_patch.py index 948fbfd53ae..9eb74aa39a0 100644 --- a/evergreen/lint_fuzzer_sanity_patch.py +++ b/evergreen/lint_fuzzer_sanity_patch.py @@ -40,31 +40,18 @@ for file in diffed_files: os.makedirs(copy_dest.parent, exist_ok=True) shutil.copy(file, copy_dest) -CONTAINER_RUNTIME = os.environ.get("CONTAINER_RUNTIME") -LOCAL_OUTPUT_FULL_DIR = Path(os.getcwd()) / OUTPUT_DIR -LOCAL_INPUT_FULL_DIR = Path(os.getcwd()) / INPUT_DIR - -CONTAINER_INPUT_PATH = f"/app/{INPUT_DIR}" -CONTAINER_OUTPUT_PATH = f"/app/{OUTPUT_DIR}" +OUTPUT_FULL_DIR = Path(os.getcwd()) / OUTPUT_DIR +INPUT_FULL_DIR = Path(os.getcwd()) / INPUT_DIR subprocess.run( [ - CONTAINER_RUNTIME, - "run", - "--rm", - "-v", - f"{LOCAL_INPUT_FULL_DIR}:{CONTAINER_INPUT_PATH}", - "-v", - f"{LOCAL_OUTPUT_FULL_DIR}:{CONTAINER_OUTPUT_PATH}", - "901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest", - "npm", - "run-script", + "./src/scripts/npm_run.sh", "jstestfuzz", "--", "--jsTestsDir", - CONTAINER_INPUT_PATH, + INPUT_FULL_DIR, "--out", - CONTAINER_OUTPUT_PATH, + OUTPUT_FULL_DIR, "--numSourceFiles", str(min(num_changed_files, 100)), "--numGeneratedFiles", @@ -76,6 +63,7 @@ subprocess.run( "10", ], check=True, + cwd="jstestfuzz", ) @@ -85,27 +73,13 @@ def _parse_jsfile(jsfile: Path) -> simple_report.Result: Returns what should be added to the report given to evergreen """ - # Find the relative path to the jsfile in the volume on the container - relative_js_file_path = f"{OUTPUT_DIR}/{jsfile.relative_to(LOCAL_OUTPUT_FULL_DIR)}" + print(f"Trying to parse jsfile {jsfile}") start_time = time.time() proc = subprocess.run( - [ - CONTAINER_RUNTIME, - "run", - "--rm", - "-v", - f"{LOCAL_INPUT_FULL_DIR}:{CONTAINER_INPUT_PATH}", - "-v", - f"{LOCAL_OUTPUT_FULL_DIR}:{CONTAINER_OUTPUT_PATH}", - "901841024863.dkr.ecr.us-east-1.amazonaws.com/mongodb-internal/jstestfuzz:latest", - "npm", - "run-script", - "parse-jsfiles", - "--", - str(relative_js_file_path), - ], + ["./src/scripts/npm_run.sh", "parse-jsfiles", "--", str(jsfile)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + cwd="jstestfuzz", ) end_time = time.time() status = "pass" if proc.returncode == 0 else "fail" @@ -130,7 +104,7 @@ report = simple_report.Report(failures=0, results=[]) with futures.ThreadPoolExecutor() as executor: parse_jsfiles_futures = [ executor.submit(_parse_jsfile, Path(jsfile)) - for jsfile in glob.iglob(f"{LOCAL_OUTPUT_FULL_DIR}/**", recursive=True) + for jsfile in glob.iglob(str(OUTPUT_FULL_DIR / "**"), recursive=True) if os.path.isfile(jsfile) ]