import {Thread} from "jstests/libs/parallelTester.js";

/* global retryOnRetryableError */

// Symbol used to override the constructor. Please do not use this, it's only meant to aid
// in migrating the jstest corpus to proper module usage.
export const kOverrideConstructor = Symbol("overrideConstructor");

// Replica set health states
const Health = {
    UP: 1,
    DOWN: 0,
};
const kOplogName = "oplog.rs";

export class ReplSetTest {
    /**
     * Sets up a replica set. To make the set running, call {@link #startSet}, followed by {@link
     * #initiate} (and optionally, {@link #awaitSecondaryNodes} to block till the  set is fully
     * operational). Note that some of the replica start up parameters are not passed here, but to
     * the #startSet method.
     *
     * @param {Object} [opts]
     * @param {string} [opts.name='testReplSet'] Name of this replica set.
     * @param {string} [opts.host] Name of the host machine. Hostname will be used if not specified.
     * @param {boolean} [opts.useHostName] If true, use hostname of machine, otherwise use
     *     localhost.
     * @param {number|Object|Object[]} [opts.nodes=0] Number of replicas.
     *     Can also be an Object (or Array).
     *     Format for Object:
     *         {
     *            <any string>: replica member option Object. see MongoRunner.runMongod
     *            <any string2>: and so on...
     *          }
     *          If object has a special "rsConfig" field then those options will be used for each
     *          replica set member config options when used to initialize the replica set, or
     *          building the config with getReplSetConfig()
     *
     *     Format for Array:
     *         An array of replica member option Object. see MongoRunner.runMongod
     *
     *     Note: For both formats, a special boolean property 'arbiter' can be specified to denote
     *     a member is an arbiter.
     *
     *     Note: A special "bridgeOptions" property can be specified in both the object and array
     *     formats to configure the options for the mongobridge corresponding to that node. These
     *     options are merged with the opts.bridgeOptions options, where the node-specific options
     *     take precedence.
     * @param {Object} [opts.nodeOptions] Command-line options to apply to all nodes in the replica
     *     set. Format for Object: { cmdline-param-with-no-arg : "", param-with-arg : arg } This
     *     turns into "mongod --cmdline-param-with-no-arg --param-with-arg arg"
     * @param {boolean} [opts.causallyConsistent=false] Specifies whether the connections to the
     *     replica set nodes should be created with the 'causal consistency' flag enabled, which
     *     means they will gossip the cluster time and add readConcern afterClusterTime where
     *     applicable.
     * @param {number} [opts.oplogSize=40]
     * @param {boolean} [opts.useSeedList=false] Use the connection string format of this set as the
     *     replica set name (overrides the name property).
     * @param {string} [opts.keyFile]
     * @param {number} [opts.protocolVersion] Protocol version of replset used by the replset
     *     initiation.
     * @param {boolean} [opts.useBridge=false] If true, then a mongobridge process is started for
     *     each node in the replica set. Both the replica set configuration and the connections
     *     returned by startSet() will be references to the proxied connections.
     * @param {Object} [opts.bridgeOptions={}] Options to apply to all mongobridge processes.
     * @param {boolean} [opts.seedRandomNumberGenerator] Indicates whether the random number
     *     generator should be seeded when randomBinVersions is true. For ReplSetTests started by
     *     ShardingTest, the seed is generated as part of ShardingTest.
     * @param {boolean} [opts.useAutoBootstrapProcedure] If true, follow the procedure for
     *     auto-bootstrapped replica sets.
     * @param {boolean} [opts.useMaintenancePorts=false] If true, then a maintenance port will be
     *     opened for each node in the replica set.
     * @param {number} [opts.timeoutMS] Timeout value in milliseconds.
     */
    constructor(opts) {
        if (this.constructor === ReplSetTest && this.constructor[kOverrideConstructor]) {
            return new this.constructor[kOverrideConstructor][kOverrideConstructor](opts);
        }

        // If opts.timeoutMS is present use that for the ReplSetTest instance, otherwise use global
        // value.
        this.timeoutMS = opts.timeoutMS || ReplSetTest.kDefaultTimeoutMS;

        // If opts is passed in as a string, let it pass unmodified since strings are pass-by-value.
        // if it is an object, though, pass in a deep copy.
        if (typeof opts === "string" || opts instanceof String) {
            retryOnRetryableError(
                () => {
                    // The primary may unexpectedly step down during startup if under heavy load
                    // and too slowly processing heartbeats. When it steps down, it closes all of
                    // its connections.
                    _constructFromExistingSeedNode(this, opts);
                },
                ReplSetTest.kDefaultRetries,
                1000,
                [ErrorCodes.NotYetInitialized],
            );
        } else if (typeof opts.rstArgs === "object") {
            _constructFromExistingNodes(this, Object.extend({}, opts.rstArgs, true));
        } else {
            _constructStartNewInstances(this, Object.extend({}, opts, true));
        }
    }

    asCluster(conn, fn, keyFileParam = undefined) {
        return asCluster(this, conn, fn, keyFileParam);
    }

    /**
     * Wait for a rs indicator to go to a particular state or states.
     *
     * @private
     * @param node is a single node, by id or conn
     * @param states is a single state or list of states
     * @param ind is the indicator specified
     * @param timeout how long to wait for the state to be reached
     * @param reconnectNode indicates that we should reconnect to a node that stepped down
     */
    _waitForIndicator(node, ind, states, timeout, reconnectNode) {
        node = resolveToConnection(this, node);
        timeout = timeout || this.timeoutMS;
        if (reconnectNode === undefined) {
            reconnectNode = true;
        }

        if (!states.length) {
            states = [states];
        }

        jsTest.log.info("ReplSetTest waitForIndicator " + ind + " on " + node);
        jsTest.log.info({states});
        jsTest.log.info("ReplSetTest waitForIndicator from node " + node);

        let lastTime = null;
        let currTime = new Date().getTime();
        let status;

        let foundState;
        assert.soon(
            () => {
                try {
                    let conn = _callHello(this);
                    if (!conn) {
                        conn = this._liveNodes[0];
                    }

                    // Try again to load connection
                    if (!conn) return false;

                    if (reconnectNode instanceof Function) {
                        // Allow caller to perform tasks on reconnect.
                        reconnectNode(conn);
                    }
                    asCluster(this, conn, function () {
                        status = conn.getDB("admin").runCommand({replSetGetStatus: 1});
                    });
                } catch (ex) {
                    jsTest.log.info("ReplSetTest waitForIndicator could not get status", {error: ex});
                    return false;
                }

                if (status.code == ErrorCodes.Unauthorized) {
                    // If we're not authorized already, then we never will be.
                    assert.commandWorked(status); // throws
                }

                let printStatus = false;
                if (lastTime == null || (currTime = new Date().getTime()) - 1000 * 5 > lastTime) {
                    if (lastTime == null) {
                        jsTest.log.info("ReplSetTest waitForIndicator Initial status (timeout : " + timeout + ") :");
                    }

                    jsTest.log.info({status});
                    lastTime = new Date().getTime();
                    printStatus = true;
                }

                if (typeof status.members == "undefined") {
                    return false;
                }

                for (let i = 0; i < status.members.length; i++) {
                    if (printStatus) {
                        jsTest.log.info(
                            "Status for : " + status.members[i].name + ", checking " + node.host + "/" + node.name,
                        );
                    }

                    if (status.members[i].name == node.host || status.members[i].name == node.name) {
                        for (let j = 0; j < states.length; j++) {
                            if (printStatus) {
                                jsTest.log.info(
                                    "Status -- " +
                                        " current state: " +
                                        status.members[i][ind] +
                                        ",  target state : " +
                                        states[j],
                                );
                            }

                            if (typeof states[j] != "number") {
                                throw new Error(
                                    "State was not an number -- type:" + typeof states[j] + ", value:" + states[j],
                                );
                            }
                            if (status.members[i][ind] == states[j]) {
                                foundState = states[j];
                                return true;
                            }
                        }
                    }
                }

                return false;
            },
            "waiting for state indicator " + ind + " for " + timeout + "ms",
            timeout,
        );

        // If we were waiting for the node to step down, wait until we can connect to it again,
        // since primaries close external connections upon stepdown. This ensures that the
        // connection to the node is usable after the function returns.
        if (reconnectNode && foundState === ReplSetTest.State.SECONDARY) {
            assert.soon(function () {
                try {
                    node.getDB("foo").bar.stats();
                    return true;
                } catch (e) {
                    return false;
                }
            }, "timed out waiting to reconnect to node " + node.name);
        }

        jsTest.log.info("ReplSetTest waitForIndicator final status:");
        jsTest.log.info({status});
    }

    /**
     * Returns the {readConcern: majority} OpTime for the host.
     * This is the OpTime of the host's "majority committed" snapshot.
     * This function may return an OpTime with Timestamp(0,0) and Term(0) if read concern majority
     * is not enabled, or if there has not been a committed snapshot yet.
     */
    getReadConcernMajorityOpTime(conn) {
        const replSetStatus = asCluster(this, conn, () =>
            assert.commandWorked(conn.getDB("admin").runCommand({replSetGetStatus: 1})),
        );

        return (
            (replSetStatus.OpTimes || replSetStatus.optimes).readConcernMajorityOpTime || {
                ts: Timestamp(0, 0),
                t: NumberLong(0),
            }
        );
    }

    /**
     * Returns the {readConcern: majority} OpTime for the host. Throws if not available.
     */
    getReadConcernMajorityOpTimeOrThrow(conn) {
        const majorityOpTime = this.getReadConcernMajorityOpTime(conn);
        if (friendlyEqual(majorityOpTime, {ts: Timestamp(0, 0), t: NumberLong(0)})) {
            throw new Error("readConcern majority optime not available");
        }
        return majorityOpTime;
    }

    /**
     * Returns list of nodes as host:port strings.
     */
    nodeList() {
        let list = [];
        for (let i = 0; i < this.ports.length; i++) {
            list.push(this.host + ":" + this.ports[i]);
        }

        return list;
    }

    getNodeId(node) {
        if (node.toFixed) {
            return parseInt(node);
        }

        for (var i = 0; i < this.nodes.length; i++) {
            if (this.nodes[i] == node) {
                return i;
            }
        }

        if (node instanceof ObjectId) {
            for (i = 0; i < this.nodes.length; i++) {
                if (this.nodes[i].runId == node) {
                    return i;
                }
            }
        }

        if (node.nodeId != null) {
            return parseInt(node.nodeId);
        }

        return undefined;
    }

    getPort(n) {
        var n = this.getNodeId(n);
        return this.ports[n];
    }

    getMaintenancePort(n) {
        const translatedN = this.getNodeId(n);
        if (this._maintenancePorts?.[translatedN] < 1) {
            throw new Error("Maintenance port not set for node");
        }
        return this._maintenancePorts[translatedN];
    }

    getNewConnectionToMaintenancePort(node) {
        const maintenancePort = this.getMaintenancePort(node);
        return new Mongo(node.host.split(":")[0] + ":" + maintenancePort);
    }

    getNewConnectionToMaintenanceSocket(node) {
        const maintenancePort = this.getMaintenancePort(node);
        return new Mongo("/tmp/mongodb-" + maintenancePort + ".sock");
    }

    getDbPath(node) {
        // Get a replica set node (check for use of bridge).
        const n = this.getNodeId(node);
        const replNode = this._useBridge ? this._unbridgedNodes[n] : this.nodes[n];
        return replNode.dbpath;
    }

    /** @private */
    _addPath(p) {
        if (!this._alldbpaths) this._alldbpaths = [p];
        else this._alldbpaths.push(p);

        return p;
    }

    // TODO (SERVER-112863): The ignoreMaintenancePort parameter is used to allow sharded clusters
    // to initiate without the maintenance port and then reconfigure with it since shard servers
    // start up with FCV lastLTS in which the maintenance port is not allowed. This can be removed
    // once sharded clusters can initiate with the maintenance port.
    getReplSetConfig(ignoreMaintenancePort = false) {
        let cfg = {};
        cfg._id = this.name;
        cfg.protocolVersion = 1;

        cfg.members = [];

        for (let i = 0; i < this.ports.length; i++) {
            let member = {};
            member._id = i;

            member.host = this.host;
            if (!member.host.includes("/")) {
                member.host += ":" + this.ports[i];
            }

            if (!ignoreMaintenancePort && this._maintenancePorts?.[i] > 0) {
                member.maintenancePort = this._maintenancePorts[i];
            }

            let nodeOpts = this.nodeOptions["n" + i];
            if (nodeOpts) {
                if (nodeOpts.arbiter) {
                    member.arbiterOnly = true;
                }

                if (nodeOpts.rsConfig) {
                    Object.extend(member, nodeOpts.rsConfig);
                }
            }

            cfg.members.push(member);
        }

        if (this._configSettings) {
            cfg.settings = this._configSettings;
        }

        return cfg;
    }

    getURL() {
        let hosts = [];

        // If the replica set uses mongobridge, use the hostname specified for the replica set.
        // If the hostname specified for the replica set or nodes is like 'primary', 'secondary0',
        // 'secondary1' (not 'localhost' and not an ip address (like 127.0.0.1 or
        // ip-10-122-7-63)), then this replica set is started by antithesis. In this
        // case, use the node's host for url so that the hostnames on the logs would be
        // different for each node. Otherwise, use the hostname specified for the replica set.
        for (let i = 0; i < this.ports.length; i++) {
            if (!this._useBridge && this.host !== "localhost" && !this.host.includes("-") && !this.host.includes(".")) {
                hosts.push(this.nodes[i].host);
            } else {
                hosts.push(this.host + ":" + this.ports[i]);
            }
        }

        return this.name + "/" + hosts.join(",");
    }

    /**
     * Starts each node in the replica set with the given options.
     *
     * @param options - The options passed to {@link MongoRunner.runMongod}
     * @param restart - Boolean indicating whether we are restarting the set (if true,
     *     then `forRestart` should have been passed as true to `stopSet()`.) Defaults to false.
     * @param skipStepUpOnRestart - Boolean indicating that this method should skip attempting to
     *     step up a new primary after restarting the set. Defaults to false. This must be set to
     *     true when using the in-memory storage engine, as the replica set must be re-initiated
     *     by the test on restart before a node can be elected.
     *     This option has no effect if `restart` is not also passed as true.
     */
    startSet(options, restart, skipStepUpOnRestart) {
        // If the caller has explicitly specified 'waitForConnect:false', then we will start up all
        // replica set nodes and return without waiting to connect to any of them.
        const skipWaitingForAllConnections = options && options.waitForConnect === false;

        // Keep a copy of these options
        this.startSetOptions = options;

        // Start up without waiting for connections.
        this.startSetAsync(options, restart);

        // Avoid waiting for connections to each node.
        if (skipWaitingForAllConnections) {
            jsTest.log.info(
                "ReplSetTest startSet skipping waiting for connections to all nodes in set '" + this.name + "'",
            );
            return this.nodes;
        }

        this.startSetAwait();

        // If the set is being restarted, by default we will try to find a node to step up
        // proactively rather than waiting for the election timeout.
        const triggerStepUp = (restart || (options && options.restart)) && !skipStepUpOnRestart;
        if (!triggerStepUp) {
            jsTest.log.info("ReplSetTest startSet skipping stepping a new primary");
            return this.nodes;
        }

        jsTest.log.info("ReplSetTest startSet attempting to step up a new primary");

        // Try to step up each node and stop after the first success.
        // We use asCluster as replSetStepUp requires auth.
        return asCluster(this, this.nodes, () => {
            for (const node of this.nodes) {
                if (_isElectable(node)) {
                    this.stepUp(node, {awaitReplicationBeforeStepUp: false});
                    return this.nodes;
                }
            }
            throw Error("Restarted set but failed to get a node to step up, as none were electable");
        });
    }

    /**
     * Starts each node in the replica set with the given options without waiting for a connection
     * to any node. Call 'startSetAwait' subsequently to wait for startup of each node to complete.
     *
     * @param options - The options passed to {@link MongoRunner.runMongod}
     */
    startSetAsync(options, restart) {
        jsTest.log.info("ReplSetTest starting set '" + this.name + "'");
        this.startSetStartTime = new Date(); // Measure the execution time of node startup.

        if (options && options.keyFile) {
            this.keyFile = options.keyFile;
        }

        if (options) {
            this.startOptions = options;
        }

        if (jsTest.options().useRandomBinVersionsWithinReplicaSet && this.seedRandomNumberGenerator) {
            // Set the random seed to the value passed in by TestData. The seed is undefined
            // by default. For sharded clusters, the seed is already initialized as part of
            // ShardingTest.
            Random.setRandomFixtureSeed();
        }

        // If the caller has explicitly set 'waitForConnect', then we prefer that. Otherwise we
        // default to not waiting for a connection. We merge the options object with a new field so
        // as to not modify the original options object that was passed in.
        options = options || {};
        options = options.waitForConnect === undefined ? Object.merge(options, {waitForConnect: false}) : options;

        // Start up each node without waiting to connect. This allows startup of replica set nodes
        // to proceed in parallel.
        for (let n = 0; n < this.ports.length; n++) {
            if (n == 0 && this.useAutoBootstrapProcedure && !this._hasAcquiredAutoGeneratedName) {
                // Must wait for connect in order to extract the auto-generated replica set name.
                options.waitForConnect = true;
            }

            this.start(n, options, restart, false);
        }
        return this.nodes;
    }

    /**
     * Waits for startup of each replica set node to complete by waiting until a connection can be
     * made to each.
     */
    startSetAwait() {
        // Wait until we can establish a connection to each node before proceeding.
        for (let n = 0; n < this.ports.length; n++) {
            this._waitForInitialConnection(n);
        }

        jsTest.log.info("ReplSetTest startSet", {nodes: this.nodes});

        jsTest.log.info(
            "ReplSetTest startSet took " +
                (new Date() - this.startSetStartTime) +
                "ms for " +
                this.nodes.length +
                " nodes.",
        );
        return this.nodes;
    }

    /**
     * Blocks until the secondary nodes have completed recovery and their roles are known. Blocks on
     * all secondary nodes or just 'secondaries', if specified. Does not wait for all 'newlyAdded'
     * fields to be removed by default.
     */
    awaitSecondaryNodes(timeout, secondaries, retryIntervalMS, waitForNewlyAddedRemoval) {
        timeout = timeout || this.timeoutMS;
        retryIntervalMS = retryIntervalMS || 200;
        let awaitingSecondaries;
        jsTest.log.info("AwaitSecondaryNodes: Waiting for the secondary nodes has started.");
        try {
            assert.soonNoExcept(
                () => {
                    awaitingSecondaries = [];
                    // Reload who the current secondaries are
                    _callHello(this);

                    let secondariesToCheck = secondaries || this._secondaries;
                    let len = secondariesToCheck.length;
                    for (let i = 0; i < len; i++) {
                        let hello = secondariesToCheck[i].getDB("admin")._helloOrLegacyHello();
                        let arbiter = hello.arbiterOnly === undefined ? false : hello.arbiterOnly;
                        if (!hello.secondary && !arbiter) {
                            awaitingSecondaries.push(secondariesToCheck[i]);
                        }
                    }

                    return awaitingSecondaries.length == 0;
                },
                "Awaiting secondaries: awaitingSecondariesPlaceholder",
                timeout,
                retryIntervalMS,
            );
        } catch (e) {
            e.message = e.message.replace(
                "awaitingSecondariesPlaceholder",
                tojson(awaitingSecondaries.map((n) => n.name)),
            );
            throw e;
        }

        // We can only wait for newlyAdded field removal if test commands are enabled.
        if (waitForNewlyAddedRemoval && jsTest.options().enableTestCommands) {
            this.waitForAllNewlyAddedRemovals();
        }
        jsTest.log.info("AwaitSecondaryNodes: Completed successfully.");
    }

    /**
     * A special version of awaitSecondaryNodes() used exclusively by rollback_test.js.
     * Wraps around awaitSecondaryNodes() itself and checks for an unrecoverable rollback
     * if it throws.
     */
    awaitSecondaryNodesForRollbackTest(timeout, secondaries, connToCheckForUnrecoverableRollback, retryIntervalMS) {
        retryIntervalMS = retryIntervalMS || 200;
        try {
            MongoRunner.runHangAnalyzer.disable();
            this.awaitSecondaryNodes(timeout, secondaries, retryIntervalMS);
            MongoRunner.runHangAnalyzer.enable();
        } catch (originalEx) {
            // Re-throw the original exception in all cases.
            MongoRunner.runHangAnalyzer.enable();
            throw originalEx;
        }
    }

    /**
     * Blocks until the specified node says it's syncing from the given upstream node.
     */
    awaitSyncSource(node, upstreamNode, timeout) {
        jsTest.log.info("Waiting for node " + node.name + " to start syncing from " + upstreamNode.name);
        let status = null;
        assert(this !== undefined);
        assert.soonNoExcept(
            function () {
                status = asCluster(this, node, () => assert.commandWorked(node.adminCommand({replSetGetStatus: 1})));

                for (let j = 0; j < status.members.length; j++) {
                    if (status.members[j].self) {
                        return status.members[j].syncSourceHost === upstreamNode.host;
                    }
                }
                return false;
            },
            "Awaiting node " + node + " syncing from " + upstreamNode + ": " + tojson(status),
            timeout,
        );
    }

    /**
     * Blocks until each node agrees that all other nodes have applied the most recent oplog entry.
     */
    awaitNodesAgreeOnAppliedOpTime(timeout, nodes) {
        timeout = timeout || this.timeoutMS;
        nodes = nodes || this.nodes;

        assert.soon(
            function () {
                let appliedOpTimeConsensus = undefined;
                for (let i = 0; i < nodes.length; i++) {
                    let replSetGetStatus;
                    try {
                        replSetGetStatus = nodes[i].adminCommand({replSetGetStatus: 1});
                    } catch (e) {
                        jsTest.log.info(
                            "AwaitNodesAgreeOnAppliedOpTime: Retrying because node " +
                                nodes[i].name +
                                " failed to execute replSetGetStatus",
                            {error: e},
                        );
                        return false;
                    }
                    assert.commandWorked(replSetGetStatus);

                    if (appliedOpTimeConsensus === undefined) {
                        if (replSetGetStatus.optimes) {
                            appliedOpTimeConsensus = replSetGetStatus.optimes.appliedOpTime;
                        } else {
                            // Older versions of mongod do not include an 'optimes' field in the
                            // replSetGetStatus response. We instead pull an optime from the first
                            // replica set member that includes one in its status. All we need here is
                            // any initial value that we can compare to all the other optimes.
                            let optimeMembers = replSetGetStatus.members.filter((m) => m.optime);
                            assert(
                                optimeMembers.length > 0,
                                "AwaitNodesAgreeOnAppliedOpTime: replSetGetStatus did not " +
                                    "include optimes for any members: " +
                                    tojson(replSetGetStatus),
                            );
                            appliedOpTimeConsensus = optimeMembers[0].optime;
                        }

                        assert(
                            appliedOpTimeConsensus,
                            "AwaitNodesAgreeOnAppliedOpTime: missing appliedOpTime in " +
                                "replSetGetStatus: " +
                                tojson(replSetGetStatus),
                        );
                    }

                    if (
                        replSetGetStatus.optimes &&
                        !friendlyEqual(replSetGetStatus.optimes.appliedOpTime, appliedOpTimeConsensus)
                    ) {
                        jsTest.log.info(
                            "AwaitNodesAgreeOnAppliedOpTime: Retrying because node " +
                                nodes[i].name +
                                " has appliedOpTime that does not match previously observed appliedOpTime",
                            {appliedOpTime: appliedOpTimeConsensus},
                        );
                        return false;
                    }

                    for (let j = 0; j < replSetGetStatus.members.length; j++) {
                        if (replSetGetStatus.members[j].state == ReplSetTest.State.ARBITER) {
                            // ARBITER nodes do not apply oplog entries and do not have an 'optime'
                            // field.
                            continue;
                        }

                        if (!friendlyEqual(replSetGetStatus.members[j].optime, appliedOpTimeConsensus)) {
                            jsTest.log.info(
                                "AwaitNodesAgreeOnAppliedOpTime: Retrying because node " +
                                    nodes[i].name +
                                    " sees optime not expected on node " +
                                    replSetGetStatus.members[j].name,
                                {
                                    actualOpTime: replSetGetStatus.members[j].optime,
                                    expectedOpTime: appliedOpTimeConsensus,
                                },
                            );
                            return false;
                        }
                    }
                }

                jsTest.log.info(
                    "AwaitNodesAgreeOnAppliedOpTime: All nodes agree that all ops are applied across replica set",
                    {appliedOpTimeConsensus},
                );
                return true;
            },
            "Awaiting nodes to agree that all ops are applied across replica set",
            timeout,
        );
    }

    /** @private */
    _findHighestPriorityNodes(config) {
        let highestPriority = 0;
        let highPriorityNodes = [];
        for (let i = 0; i < config.members.length; i++) {
            const member = config.members[i];
            if (member.priority > highestPriority) {
                highestPriority = member.priority;
                highPriorityNodes = [this.nodes[i]];
            } else if (member.priority === highestPriority) {
                highPriorityNodes.push(this.nodes[i]);
            }
        }
        return highPriorityNodes;
    }

    /**
     * Blocks until the node with the highest priority is the primary.  If there are multiple
     * nodes tied for highest priority, waits until one of them is the primary.
     */
    awaitHighestPriorityNodeIsPrimary(timeout) {
        timeout = timeout || this.timeoutMS;

        // First figure out the set of highest priority nodes.
        const config = asCluster(this, this.nodes, () => this.getReplSetConfigFromNode());
        const highPriorityNodes = this._findHighestPriorityNodes(config);

        // Now wait for the primary to be one of the highest priority nodes.
        assert.soon(
            () => {
                return highPriorityNodes.includes(this.getPrimary());
            },
            () => {
                return (
                    "Expected primary to be one of: " +
                    tojson(highPriorityNodes) +
                    ", but found primary to be: " +
                    tojson(this.getPrimary())
                );
            },
            timeout,
        );

        // Finally wait for all nodes to agree on the primary.
        this.awaitNodesAgreeOnPrimary(timeout);
        const primary = this.getPrimary();
        assert(
            highPriorityNodes.includes(primary),
            "Primary switched away from highest priority node.  Found primary: " +
                tojson(primary) +
                ", but expected one of: " +
                tojson(highPriorityNodes),
        );
    }

    /**
     * Blocks until all nodes agree on who the primary is.
     * Unlike awaitNodesAgreeOnPrimary, this does not require that all nodes are authenticated.
     */
    awaitNodesAgreeOnPrimaryNoAuth(timeout, nodes) {
        timeout = timeout || this.timeoutMS;
        nodes = nodes || this.nodes;

        jsTest.log.info("AwaitNodesAgreeOnPrimaryNoAuth: Waiting for nodes to agree on any primary.");

        assert.soonNoExcept(
            function () {
                let primary;

                for (let i = 0; i < nodes.length; i++) {
                    let hello = assert.commandWorked(nodes[i].getDB("admin")._helloOrLegacyHello());
                    let nodesPrimary = hello.primary;
                    // Node doesn't see a primary.
                    if (!nodesPrimary) {
                        jsTest.log.info(
                            "AwaitNodesAgreeOnPrimaryNoAuth: Retrying because " +
                                nodes[i].name +
                                " does not see a primary.",
                        );
                        return false;
                    }

                    if (!primary) {
                        // If we haven't seen a primary yet, set it to this.
                        primary = nodesPrimary;
                    } else if (primary !== nodesPrimary) {
                        jsTest.log.info(
                            "AwaitNodesAgreeOnPrimaryNoAuth: Retrying because " +
                                nodes[i].name +
                                " thinks the primary is " +
                                nodesPrimary +
                                " instead of " +
                                primary,
                        );
                        return false;
                    }
                }

                jsTest.log.info("AwaitNodesAgreeOnPrimaryNoAuth: Nodes agreed on primary " + primary);
                return true;
            },
            "Awaiting nodes to agree on primary",
            timeout,
        );
    }

    /**
     * Blocks until all nodes agree on who the primary is.
     * If 'expectedPrimaryNode' is provided, ensure that every node is seeing this node as the
     * primary. Otherwise, ensure that all the nodes in the set agree with the first node on the
     * identity of the primary.
     * This call does not guarantee that the agreed upon primary is writeable.
     */
    awaitNodesAgreeOnPrimary(timeout, nodes, expectedPrimaryNode, runHangAnalyzerOnTimeout = true) {
        timeout = timeout || this.timeoutMS;
        nodes = nodes || this.nodes;
        // indexOf will return the index of the expected node. If expectedPrimaryNode is undefined,
        // indexOf will return -1.
        const expectedPrimaryNodeIdx = this.nodes.indexOf(expectedPrimaryNode);
        if (expectedPrimaryNodeIdx === -1) {
            jsTest.log.info("AwaitNodesAgreeOnPrimary: Waiting for nodes to agree on any primary.");
        } else {
            jsTest.log.info(
                "AwaitNodesAgreeOnPrimary: Waiting for nodes to agree on " + expectedPrimaryNode.name + " as primary.",
            );
        }

        assert.soonNoExcept(
            () => {
                let primary = expectedPrimaryNodeIdx;

                for (let i = 0; i < nodes.length; i++) {
                    let node = nodes[i];
                    let replSetGetStatus = asCluster(this, node, () =>
                        assert.commandWorked(node.adminCommand({replSetGetStatus: 1})),
                    );
                    let nodesPrimary = -1;
                    for (let j = 0; j < replSetGetStatus.members.length; j++) {
                        if (replSetGetStatus.members[j].state === ReplSetTest.State.PRIMARY) {
                            // Node sees two primaries.
                            if (nodesPrimary !== -1) {
                                jsTest.log.info(
                                    "AwaitNodesAgreeOnPrimary: Retrying because " +
                                        nodes[i].name +
                                        " thinks both " +
                                        this.nodes[nodesPrimary].name +
                                        " and " +
                                        this.nodes[j].name +
                                        " are primary.",
                                );

                                return false;
                            }
                            nodesPrimary = j;
                        }
                    }
                    // Node doesn't see a primary.
                    if (nodesPrimary < 0) {
                        jsTest.log.info(
                            "AwaitNodesAgreeOnPrimary: Retrying because " + node.name + " does not see a primary.",
                        );
                        return false;
                    }

                    if (primary < 0) {
                        jsTest.log.info(
                            "AwaitNodesAgreeOnPrimary: " +
                                node.name +
                                " thinks the " +
                                " primary is " +
                                this.nodes[nodesPrimary].name +
                                ". Other nodes are expected to agree on the same primary.",
                        );
                        // If the nodes haven't seen a primary yet, set primary to nodes[i]'s primary.
                        primary = nodesPrimary;
                    } else if (primary !== nodesPrimary) {
                        jsTest.log.info(
                            "AwaitNodesAgreeOnPrimary: Retrying because " +
                                node.name +
                                " thinks the primary is " +
                                this.nodes[nodesPrimary].name +
                                " instead of " +
                                this.nodes[primary].name,
                        );
                        return false;
                    }
                }

                jsTest.log.info("AwaitNodesAgreeOnPrimary: Nodes agreed on primary " + this.nodes[primary].name);
                return true;
            },
            "Awaiting nodes to agree on primary timed out",
            timeout,
            undefined /*interval*/,
            {
                runHangAnalyzer: runHangAnalyzerOnTimeout,
            },
        );
    }

    /**
     * Blocking call, which will wait for a primary to be elected and become writable for some
     * pre-defined timeout. If a primary is available it will return a connection to it.
     * Otherwise throws an exception.
     */
    getPrimary(timeout, retryIntervalMS) {
        timeout = timeout || this.timeoutMS;
        retryIntervalMS = retryIntervalMS || 200;
        let primary = null;

        assert.soonNoExcept(
            () => {
                primary = _callHello(this);
                return primary;
            },
            "Finding primary",
            timeout,
            retryIntervalMS,
        );

        return primary;
    }

    /**
     * Blocks until all nodes agree on who the primary is and the primary is writeable.
     * This includes waiting for the optional primary catchup process to complete, which
     * getPrimary() ensures.
     * Returns the primary.
     */
    awaitNodesAgreeOnWriteablePrimary(
        expectedPrimaryNode,
        waitPrimaryWriteTimeout,
        retryIntervalMS,
        waitNodesAgreeTimeout,
        nodes,
        runHangAnalyzerOnTimeout = true,
    ) {
        this.awaitNodesAgreeOnPrimary(waitNodesAgreeTimeout, nodes, expectedPrimaryNode, runHangAnalyzerOnTimeout);
        return this.getPrimary(waitPrimaryWriteTimeout, retryIntervalMS);
    }

    awaitNoPrimary(msg, timeout) {
        msg = msg || "Timed out waiting for there to be no primary in replset: " + this.name;
        timeout = timeout || this.timeoutMS;

        assert.soonNoExcept(
            () => {
                return _callHello(this) == false;
            },
            msg,
            timeout,
        );
    }

    getSecondaries(timeout) {
        let primary = this.getPrimary(timeout);
        let secs = [];
        for (let i = 0; i < this.nodes.length; i++) {
            if (this.nodes[i] != primary) {
                secs.push(this.nodes[i]);
            }
        }

        return secs;
    }

    getSecondary(timeout) {
        return this.getSecondaries(timeout)[0];
    }

    getArbiters() {
        let arbiters = [];
        for (let i = 0; i < this.nodes.length; i++) {
            const node = this.nodes[i];

            let isArbiter = false;

            assert.retryNoExcept(
                () => {
                    isArbiter = isNodeArbiter(node);
                    return true;
                },
                `Could not call hello/isMaster on ${node}.`,
                3,
                1000,
            );

            if (isArbiter) {
                arbiters.push(node);
            }
        }
        return arbiters;
    }

    getArbiter() {
        return this.getArbiters()[0];
    }

    status(timeout) {
        let primary = _callHello(this);
        if (!primary) {
            primary = this._liveNodes[0];
        }
        return asCluster(this, primary, () => assert.commandWorked(primary.adminCommand({replSetGetStatus: 1})));
    }

    /**
     * Adds a node to the replica set managed by this instance.
     */
    add(config) {
        let nextPort = this._allocatePortForNode();
        jsTest.log.info("ReplSetTest Next port: " + nextPort);

        this.ports.push(nextPort);
        jsTest.log.info({ports: this.ports});

        if (this._useBridge) {
            this._unbridgedPorts.push(this._allocatePortForBridge());
        }

        if (this._useMaintenancePorts) {
            this._maintenancePorts.push(this._allocatePortForNode());
        }

        if (jsTestOptions().shellGRPC) {
            const nextPort = this._allocatePortForNode();
            jsTest.log.info("ReplSetTest Next gRPC port: " + nextPort);

            this.grpcPorts.push(nextPort);
            jsTest.log.info({grpcPorts: this.grpcPorts});
        }

        let nextId = this.nodes.length;
        jsTest.log.info({nodes: this.nodes});

        jsTest.log.info("ReplSetTest nextId: " + nextId);
        return this.start(nextId, config);
    }

    /**
     * Calls stop() on the node identifed by nodeId and removes it from the list of nodes managed by
     * ReplSetTest.
     */
    remove(nodeId) {
        this.stop(nodeId);
        nodeId = this.getNodeId(nodeId);
        this.nodes.splice(nodeId, 1);
        this.ports.splice(nodeId, 1);

        if (this._useBridge) {
            this._unbridgedPorts.splice(nodeId, 1);
            this._unbridgedNodes.splice(nodeId, 1);
        }

        if (this._maintenancePorts) {
            this._maintenancePorts.splice(nodeId, 1);
        }
    }

    /**
     * If journaling is disabled or we are using an ephemeral storage engine, set
     * 'writeConcernMajorityJournalDefault' to false for the given 'config' object. If the
     * 'writeConcernMajorityJournalDefault' field is already set, it does not override it,
     * and returns the 'config' object unchanged. Does not affect 'config' when running CSRS.
     *
     * @private
     */
    _updateConfigIfNotDurable(config) {
        // Get a replica set node (check for use of bridge).
        let replNode = this._useBridge ? this._unbridgedNodes[0] : this.nodes[0];

        // Don't update replset config for sharding config servers since config servers always
        // require durable storage.
        if (replNode.hasOwnProperty("fullOptions") && replNode.fullOptions.hasOwnProperty("configsvr")) {
            return config;
        }

        // Don't override existing value.
        let wcMajorityJournalField = "writeConcernMajorityJournalDefault";
        if (config.hasOwnProperty(wcMajorityJournalField)) {
            return config;
        }

        // Check journaling by sending commands through the bridge if it's used.
        if (_isRunningWithoutJournaling(this, this.nodes[0])) {
            config[wcMajorityJournalField] = false;
        }

        return config;
    }

    /** @private */
    _setDefaultConfigOptions(config) {
        // Update config for non journaling test variants
        this._updateConfigIfNotDurable(config);
        // Add protocolVersion if missing
        if (!config.hasOwnProperty("protocolVersion")) {
            config["protocolVersion"] = 1;
        }
    }

    /** @private */
    _notX509Auth(conn) {
        const nodeId = "n" + this.getNodeId(conn);
        const nodeOptions = this.nodeOptions[nodeId] || {};
        const options = Object.keys(nodeOptions).length !== 0 || !this.startOptions ? nodeOptions : this.startOptions;
        const authMode = options.clusterAuthMode;
        return authMode != "sendX509" && authMode != "x509" && authMode != "sendKeyFile";
    }

    /**
     * Wait until the config on the primary becomes replicated. Callers specify the primary in case
     * this must be called when two nodes are expected to be concurrently primary. This does not
     * necessarily wait for the config to be committed.
     */
    waitForConfigReplication(primary, nodes) {
        const nodeHosts = nodes ? tojson(nodes.map((n) => n.host)) : "all nodes";
        jsTest.log.info(
            "waitForConfigReplication: Waiting for the config on " + primary.host + " to replicate to " + nodeHosts,
        );

        let rst = this;
        let configVersion = -2;
        let configTerm = -2;
        asCluster(this, primary, () => {
            assert.soon(function () {
                const res = assert.commandWorked(primary.adminCommand({replSetGetStatus: 1}));
                const primaryMember = res.members.find((m) => m.self);
                configVersion = primaryMember.configVersion;
                configTerm = primaryMember.configTerm;
                function hasSameConfig(member) {
                    return (
                        member.configVersion === primaryMember.configVersion &&
                        member.configTerm === primaryMember.configTerm
                    );
                }
                let members = res.members;
                if (nodes) {
                    members = res.members.filter((m) => nodes.some((node) => m.name === node.name));
                }
                return members.every((m) => hasSameConfig(m));
            });
        });

        jsTest.log.info(
            "waitForConfigReplication: config on " +
                primary.host +
                " replicated successfully to " +
                nodeHosts +
                " with version " +
                configVersion +
                " and term " +
                configTerm,
        );
    }

    /**
     * Waits for all 'newlyAdded' fields to be removed, for that config to be committed, and for
     * the in-memory and on-disk configs to match.
     */
    waitForAllNewlyAddedRemovals(timeout) {
        timeout = timeout || this.timeoutMS;
        jsTest.log.info("waitForAllNewlyAddedRemovals: starting for set " + this.name);
        const primary = this.getPrimary();

        // Shadow 'db' so that we can call the function on the primary without a separate shell when
        // x509 auth is not needed.
        let db = primary.getDB("admin");
        runFnWithAuthOnPrimary(
            this,
            function () {
                assert.soon(function () {
                    const getConfigRes = assert.commandWorkedOrFailedWithCode(
                        db.adminCommand({
                            replSetGetConfig: 1,
                            commitmentStatus: true,
                            $_internalIncludeNewlyAdded: true,
                        }),
                        ErrorCodes.NotWritablePrimary,
                    );

                    if (!getConfigRes.ok) {
                        jsTest.log.info(
                            "waitForAllNewlyAddedRemovals: Retrying because the old primary " + " stepped down",
                        );
                        return false;
                    }

                    const config = getConfigRes.config;
                    for (let i = 0; i < config.members.length; i++) {
                        const memberConfig = config.members[i];
                        if (memberConfig.hasOwnProperty("newlyAdded")) {
                            assert(memberConfig["newlyAdded"] === true, config);
                            jsTest.log.info(
                                "waitForAllNewlyAddedRemovals: Retrying because memberIndex " +
                                    i +
                                    " is still 'newlyAdded'",
                            );
                            return false;
                        }
                    }
                    if (!getConfigRes.hasOwnProperty("commitmentStatus")) {
                        jsTest.log.info(
                            "waitForAllNewlyAddedRemovals: Skipping wait due to no commitmentStatus." +
                                " Assuming this is an older version.",
                        );
                        return true;
                    }

                    if (!getConfigRes.commitmentStatus) {
                        jsTest.log.info(
                            "waitForAllNewlyAddedRemovals: " +
                                "Retrying because primary's config isn't committed. " +
                                "Version: " +
                                config.version +
                                ", Term: " +
                                config.term,
                        );
                        return false;
                    }

                    return true;
                });
            },
            "waitForAllNewlyAddedRemovals",
        );

        this.waitForConfigReplication(primary);

        jsTest.log.info("waitForAllNewlyAddedRemovals: finished for set " + this.name);
    }

    /**
     * Runs replSetInitiate on the first node of the replica set.
     * Ensures that a primary is elected (not necessarily node 0).
     * initiate() should be preferred instead of this, but this is useful when the connections
     * aren't authorized to run replSetGetStatus.
     */
    _initiateWithAnyNodeAsPrimary(
        cfg,
        initCmd,
        {
            doNotWaitForStableRecoveryTimestamp: doNotWaitForStableRecoveryTimestamp = false,
            doNotWaitForReplication: doNotWaitForReplication = false,
            doNotWaitForNewlyAddedRemovals: doNotWaitForNewlyAddedRemovals = false,
            doNotWaitForPrimaryOnlyServices: doNotWaitForPrimaryOnlyServices = false,
        } = {},
    ) {
        let startTime = new Date(); // Measure the execution time of this function.
        let primary = this.nodes[0].getDB("admin");
        let config = cfg || this.getReplSetConfig();
        let cmd = {};
        let cmdKey = initCmd || "replSetInitiate";

        // Throw an exception if nodes[0] is unelectable in the given config.
        if (!_isElectable(config.members[0])) {
            throw Error("The node at index 0 must be electable");
        }

        // Start up a single node replica set then reconfigure to the correct size (if the config
        // contains more than 1 node), so the primary is elected more quickly.
        let originalMembers, originalSettings;
        if (config.members && config.members.length > 1) {
            originalMembers = config.members.slice();
            config.members = config.members.slice(0, 1);
            originalSettings = config.settings;
            delete config.settings; // Clear settings to avoid tags referencing sliced nodes.
        }
        this._setDefaultConfigOptions(config);

        cmd[cmdKey] = config;

        // If this ReplSet is started using this.startSet and binVersions (ie:
        // rst.startSet({binVersion: [...]}) we need to make sure the binVersion combination is
        // valid.
        if (
            typeof this.startSetOptions === "object" &&
            this.startSetOptions.hasOwnProperty("binVersion") &&
            typeof this.startSetOptions.binVersion === "object"
        ) {
            let lastLTSSpecified = false;
            let lastContinuousSpecified = false;
            this.startSetOptions.binVersion.forEach(function (binVersion, _) {
                if (lastLTSSpecified === false) {
                    lastLTSSpecified = MongoRunner.areBinVersionsTheSame(binVersion, lastLTSFCV);
                }
                if (lastContinuousSpecified === false && lastLTSFCV !== lastContinuousFCV) {
                    lastContinuousSpecified = MongoRunner.areBinVersionsTheSame(binVersion, lastContinuousFCV);
                }
            });
            if (lastLTSSpecified && lastContinuousSpecified) {
                throw new Error(
                    "Can only specify one of 'last-lts' and 'last-continuous' " + "in binVersion, not both.",
                );
            }
        }
        // Initiating a replica set with a single node will use "latest" FCV. This will
        // cause IncompatibleServerVersion errors if additional "last-lts"/"last-continuous" binary
        // version nodes are subsequently added to the set, since such nodes cannot set their FCV to
        // "latest". Therefore, we make sure the primary is "last-lts"/"last-continuous" FCV before
        // adding in nodes of different binary versions to the replica set.
        let lastLTSBinVersionWasSpecifiedForSomeNode = false;
        let lastContinuousBinVersionWasSpecifiedForSomeNode = false;
        let explicitBinVersionWasSpecifiedForSomeNode = false;
        Object.keys(this.nodeOptions).forEach((key) => {
            let val = this.nodeOptions[key];
            if (typeof val === "object" && val.hasOwnProperty("binVersion")) {
                if (lastLTSBinVersionWasSpecifiedForSomeNode === false) {
                    lastLTSBinVersionWasSpecifiedForSomeNode = MongoRunner.areBinVersionsTheSame(
                        val.binVersion,
                        lastLTSFCV,
                    );
                }
                if (lastContinuousBinVersionWasSpecifiedForSomeNode === false && lastLTSFCV !== lastContinuousFCV) {
                    lastContinuousBinVersionWasSpecifiedForSomeNode = MongoRunner.areBinVersionsTheSame(
                        val.binVersion,
                        lastContinuousFCV,
                    );
                }
                explicitBinVersionWasSpecifiedForSomeNode = true;
            }
        });

        if (lastLTSBinVersionWasSpecifiedForSomeNode && lastContinuousBinVersionWasSpecifiedForSomeNode) {
            throw new Error("Can only specify one of 'last-lts' and 'last-continuous' " + "in binVersion, not both.");
        }

        // If no binVersions have been explicitly set, then we should be using the latest binary
        // version, which allows us to use the failpoint below.
        let explicitBinVersion =
            (this.startOptions !== undefined && this.startOptions.hasOwnProperty("binVersion")) ||
            explicitBinVersionWasSpecifiedForSomeNode ||
            jsTest.options().useRandomBinVersionsWithinReplicaSet;

        // If a test has explicitly disabled test commands or if we may be running an older mongod
        // version then we cannot utilize failpoints below, since they may not be supported on older
        // versions.
        const failPointsSupported = jsTest.options().enableTestCommands && !explicitBinVersion;

        // Skip waiting for new data to appear in the oplog buffer when transitioning to primary.
        // This makes step up much faster for a node that doesn't need to drain any oplog
        // operations. This is only an optimization so it's OK if we bypass it in some suites.
        if (failPointsSupported) {
            setFailPoint(this.nodes[0], "skipOplogBatcherWaitForData");
        }

        // replSetInitiate and replSetReconfig commands can fail with a NodeNotFound error if a
        // heartbeat times out during the quorum check. They may also fail with
        // NewReplicaSetConfigurationIncompatible on similar timeout during the config validation
        // stage while deducing isSelf(). This can fail with an InterruptedDueToReplStateChange
        // error when interrupted. We try several times, to reduce the chance of failing this way.
        const initiateStart = new Date(); // Measure the execution time of this section.

        if (this.useAutoBootstrapProcedure) {
            // Auto-bootstrap already initiates automatically on the first node, but if the
            // requested initiate is not empty, we need to apply the requested settings using
            // reconfig.
            if (cmd[cmdKey] != {}) {
                cmd["replSetReconfig"] = cmd[cmdKey];
                delete cmd[cmdKey];

                // We must increase the version of the new config for the reconfig
                // to succeed. The initial default config will always have a version of 1.
                cmd["replSetReconfig"].version = 2;
                replSetCommandWithRetry(primary, cmd);
            }
        } else {
            replSetCommandWithRetry(primary, cmd);
        }

        // Blocks until there is a primary. We use a faster retry interval here since we expect the
        // primary to be ready very soon. We also turn the failpoint off once we have a primary.
        this.getPrimary(this.timeoutMS, 25 /* retryIntervalMS */);
        if (failPointsSupported) {
            clearFailPoint(this.nodes[0], "skipOplogBatcherWaitForData");
        }

        jsTest.log.info(
            "ReplSetTest initiate command took " +
                (new Date() - initiateStart) +
                "ms for " +
                this.nodes.length +
                " nodes in set '" +
                this.name +
                "'",
        );

        // Set the FCV to 'last-lts'/'last-continuous' if we are running a mixed version replica
        // set. If this is a config server, the FCV will be set as part of ShardingTest.
        // versions are supported with the useRandomBinVersionsWithinReplicaSet option.
        let setLastLTSFCV =
            (lastLTSBinVersionWasSpecifiedForSomeNode ||
                jsTest.options().useRandomBinVersionsWithinReplicaSet == "last-lts") &&
            !this.isConfigServer;
        let setLastContinuousFCV =
            !setLastLTSFCV &&
            (lastContinuousBinVersionWasSpecifiedForSomeNode ||
                jsTest.options().useRandomBinVersionsWithinReplicaSet == "last-continuous") &&
            !this.isConfigServer;

        if (setLastLTSFCV || setLastContinuousFCV) {
            // Authenticate before running the command.
            asCluster(this, this.nodes, () => {
                let fcv = setLastLTSFCV ? lastLTSFCV : lastContinuousFCV;

                jsTest.log.info("Setting feature compatibility version for replica set to '" + fcv + "'");
                // When latest is not equal to last-continuous, the transition to last-continuous is
                // not allowed. Setting fromConfigServer allows us to bypass this restriction and
                // test last-continuous.
                assert.commandWorked(
                    this.getPrimary().adminCommand({
                        setFeatureCompatibilityVersion: fcv,
                        fromConfigServer: true,
                        confirm: true,
                    }),
                );
                checkFCV(this.getPrimary().getDB("admin"), fcv);

                // The server has a practice of adding a reconfig as part of upgrade/downgrade logic
                // in the setFeatureCompatibilityVersion command.
                jsTest.log.info(
                    "Fetch the config version from primary since last-lts or last-continuous downgrade might " +
                        "perform a reconfig.",
                );
                config.version = this.getReplSetConfigFromNode().version;
            });
        }

        // Wait for 2 keys to appear before adding the other nodes. This is to prevent replica
        // set configurations from interfering with the primary to generate the keys. One example
        // of problematic configuration are delayed secondaries, which impedes the primary from
        // generating the second key due to timeout waiting for write concern.
        let shouldWaitForKeys = true;
        if (this.waitForKeys != undefined) {
            shouldWaitForKeys = this.waitForKeys;
            jsTest.log.info("Set shouldWaitForKeys from RS options: " + shouldWaitForKeys);
        } else {
            Object.keys(this.nodeOptions).forEach((key) => {
                let val = this.nodeOptions[key];
                if (
                    typeof val === "object" &&
                    (val.hasOwnProperty("shardsvr") ||
                        (val.hasOwnProperty("binVersion") &&
                            // Should not wait for keys if version is less than 3.6
                            MongoRunner.compareBinVersions(val.binVersion, "3.6") == -1))
                ) {
                    shouldWaitForKeys = false;
                    jsTest.log.info("Set shouldWaitForKeys from node options: " + shouldWaitForKeys);
                }
            });
            if (this.startOptions != undefined) {
                let val = this.startOptions;
                if (
                    typeof val === "object" &&
                    (val.hasOwnProperty("shardsvr") ||
                        (val.hasOwnProperty("binVersion") &&
                            // Should not wait for keys if version is less than 3.6
                            MongoRunner.compareBinVersions(val.binVersion, "3.6") == -1))
                ) {
                    shouldWaitForKeys = false;
                    jsTest.log.info("Set shouldWaitForKeys from start options: " + shouldWaitForKeys);
                }
            }
        }
        /**
         * Blocks until the primary node generates cluster time sign keys.
         */
        if (shouldWaitForKeys) {
            asCluster(this, this.nodes, (timeout) => {
                jsTest.log.info("Waiting for keys to sign $clusterTime to be generated");
                assert.soonNoExcept(
                    (timeout) => {
                        let keyCnt = this.getPrimary(timeout)
                            .getCollection("admin.system.keys")
                            .find({purpose: "HMAC"})
                            .itcount();
                        return keyCnt >= 2;
                    },
                    "Awaiting keys",
                    timeout,
                );
            });
        }

        // Allow nodes to find sync sources more quickly. We also turn down the heartbeat interval
        // to speed up the initiation process. We use a failpoint so that we can easily turn this
        // behavior on/off without doing a reconfig. This is only an optimization so it's OK if we
        // bypass it in some suites.
        if (failPointsSupported) {
            this.nodes.forEach(function (conn) {
                setFailPoint(conn, "forceSyncSourceRetryWaitForInitialSync", {retryMS: 25});
                setFailPoint(conn, "forceHeartbeatIntervalMS", {intervalMS: 200});
                setFailPoint(conn, "forceBgSyncSyncSourceRetryWaitMS", {sleepMS: 25});
            });
        }

        // Reconfigure the set to contain the correct number of nodes (if necessary).
        const reconfigStart = new Date(); // Measure duration of reconfig and awaitSecondaryNodes.
        if (originalMembers) {
            config.members = originalMembers;
            if (originalSettings) {
                config.settings = originalSettings;
            }
            config.version = config.version ? config.version + 1 : 2;

            // Nodes started with the --configsvr flag must have configsvr = true in their config.
            if (this.nodes[0].hasOwnProperty("fullOptions") && this.nodes[0].fullOptions.hasOwnProperty("configsvr")) {
                config.configsvr = true;
            }

            // Add in nodes 1 at a time since non-force reconfig allows only single node
            // addition/removal.
            jsTest.log.info("Reconfiguring replica set to add in other nodes");
            for (let i = 2; i <= originalMembers.length; i++) {
                jsTest.log.info("ReplSetTest adding in node " + i);
                assert.soon(
                    () => {
                        primary = this.getPrimary();
                        const statusRes = asCluster(this, primary, () =>
                            assert.commandWorked(primary.adminCommand({replSetGetStatus: 1})),
                        );
                        const primaryMember = statusRes.members.find((m) => m.self);
                        config.version = primaryMember.configVersion + 1;

                        config.members = originalMembers.slice(0, i);
                        cmd = {replSetReconfig: config, maxTimeMS: this.timeoutMS};
                        jsTest.log.info("Running reconfig command", {cmd});
                        const reconfigRes = primary.adminCommand(cmd);
                        const retryableReconfigCodes = [
                            ErrorCodes.NodeNotFound,
                            ErrorCodes.NewReplicaSetConfigurationIncompatible,
                            ErrorCodes.InterruptedDueToReplStateChange,
                            ErrorCodes.ConfigurationInProgress,
                            ErrorCodes.CurrentConfigNotCommittedYet,
                            ErrorCodes.NotWritablePrimary,
                        ];
                        if (retryableReconfigCodes.includes(reconfigRes.code)) {
                            jsTest.log.info("Retrying reconfig", {reconfigRes});
                            return false;
                        }
                        assert.commandWorked(reconfigRes);
                        return true;
                    },
                    "reconfig for fixture set up failed",
                    this.timeoutMS,
                    1000,
                );
            }
        }

        // Setup authentication if running test with authentication
        if ((jsTestOptions().keyFile || this.clusterAuthMode === "x509") && cmdKey === "replSetInitiate") {
            primary = this.getPrimary();
            // The sslSpecial suite sets up cluster with x509 but the shell was not started with TLS
            // so we need to rely on the test to auth if needed.
            if (!(this.clusterAuthMode === "x509" && !primary.isTLS())) {
                jsTest.authenticateNodes(this.nodes);
            }
        }

        // Wait for initial sync to complete on all nodes. Use a faster polling interval so we can
        // detect initial sync completion more quickly.
        this.awaitSecondaryNodes(this.timeoutMS, null /* secondaries */, 25 /* retryIntervalMS */);

        // If test commands are not enabled, we cannot wait for 'newlyAdded' removals. Tests that
        // disable test commands must ensure 'newlyAdded' removals mid-test are acceptable.
        if (!doNotWaitForNewlyAddedRemovals && jsTest.options().enableTestCommands) {
            this.waitForAllNewlyAddedRemovals();
        }

        jsTest.log.info(
            "ReplSetTest initiate reconfig and awaitSecondaryNodes took " +
                (new Date() - reconfigStart) +
                "ms for " +
                this.nodes.length +
                " nodes in set '" +
                this.name +
                "'",
        );

        try {
            this.awaitHighestPriorityNodeIsPrimary();
        } catch (e) {
            // Due to SERVER-14017, the call to awaitHighestPriorityNodeIsPrimary() may fail
            // in certain configurations due to being unauthorized.  In that case we proceed
            // even though we aren't guaranteed that the highest priority node is the one that
            // became primary.
            // TODO(SERVER-14017): Unconditionally expect awaitHighestPriorityNodeIsPrimary to pass.
            assert.eq(ErrorCodes.Unauthorized, e.code, tojson(e));
            jsTest.log.info(
                "Running awaitHighestPriorityNodeIsPrimary() during ReplSetTest initialization " +
                    "failed with Unauthorized error, proceeding even though we aren't guaranteed " +
                    "that the highest priority node is primary",
            );
        }

        // We need to disable the enableDefaultWriteConcernUpdatesForInitiate parameter
        // to disallow updating the default write concern after initiating is complete.
        asCluster(this, this.nodes, () => {
            for (let node of this.nodes) {
                // asCluster() currently does not validate connections with X509 authentication.
                // If the test is using X509, we skip disabling the server parameter as the
                // 'setParameter' command will fail.
                // TODO(SERVER-57924): cleanup asCluster() to avoid checking here.
                if (this._notX509Auth(node) || node.isTLS()) {
                    const serverStatus = assert.commandWorked(node.getDB("admin").runCommand({serverStatus: 1}));
                    const currVersion = serverStatus.version;
                    const olderThan50 =
                        MongoRunner.compareBinVersions(
                            MongoRunner.getBinVersionFor("5.0"),
                            MongoRunner.getBinVersionFor(currVersion),
                        ) === 1;

                    // The following params are available only on versions greater than or equal to
                    // 5.0.
                    if (olderThan50) {
                        continue;
                    }

                    if (jsTestOptions().enableTestCommands) {
                        assert.commandWorked(
                            node.adminCommand({
                                setParameter: 1,
                                enableDefaultWriteConcernUpdatesForInitiate: false,
                            }),
                        );

                        // Re-enable the reconfig check to ensure that committed writes cannot be rolled
                        // back. We disabled this check during initialization to ensure that replica
                        // sets will not fail to start up.
                        assert.commandWorked(
                            node.adminCommand({setParameter: 1, enableReconfigRollbackCommittedWritesCheck: true}),
                        );
                    }
                }
            }
        });

        const awaitTsStart = new Date(); // Measure duration of awaitLastStableRecoveryTimestamp.
        if (!doNotWaitForStableRecoveryTimestamp) {
            // Speed up the polling interval so we can detect recovery timestamps more quickly.
            this.awaitLastStableRecoveryTimestamp(25 /* retryIntervalMS */);
        }
        jsTest.log.info(
            "ReplSetTest initiate awaitLastStableRecoveryTimestamp took " +
                (new Date() - awaitTsStart) +
                "ms for " +
                this.nodes.length +
                " nodes in set '" +
                this.name +
                "'",
        );

        // Waits for the services which write on step-up to finish rebuilding to avoid background
        // writes after initiation is done. PrimaryOnlyServices wait for the stepup optime to be
        // majority committed before rebuilding services, so we skip waiting for PrimaryOnlyServices
        // if we do not wait for replication.
        if (!doNotWaitForReplication && !doNotWaitForPrimaryOnlyServices) {
            primary = this.getPrimary();
            // TODO(SERVER-57924): cleanup asCluster() to avoid checking here.
            if (this._notX509Auth(primary) || primary.isTLS()) {
                asCluster(this, primary, () => this.waitForStepUpWrites(primary));
            }
        }

        // Make sure all nodes are up to date. Bypass this if the heartbeat interval wasn't turned
        // down or the test specifies that we should not wait for replication. This is only an
        // optimization so it's OK if we bypass it in some suites.
        if (failPointsSupported && !doNotWaitForReplication) {
            asCluster(this, this.nodes, () => this.awaitNodesAgreeOnAppliedOpTime());
        }

        // Turn off the failpoints now that initial sync and initial setup is complete.
        if (failPointsSupported) {
            this.nodes.forEach(function (conn) {
                clearFailPoint(conn, "forceSyncSourceRetryWaitForInitialSync");
                clearFailPoint(conn, "forceHeartbeatIntervalMS");
                clearFailPoint(conn, "forceBgSyncSyncSourceRetryWaitMS");
            });
        }

        jsTest.log.info(
            "ReplSetTest initiateWithAnyNodeAsPrimary took " +
                (new Date() - startTime) +
                "ms for " +
                this.nodes.length +
                " nodes.",
        );
    }

    /**
     * Runs replSetInitiate on the replica set and requests the first node to step up as primary.
     * This version should be prefered where possible but requires all connections in the
     * ReplSetTest to be authorized to run replSetGetStatus.
     */
    _initiateWithNodeZeroAsPrimary(
        cfg,
        initCmd,
        {
            doNotWaitForStableRecoveryTimestamp: doNotWaitForStableRecoveryTimestamp = false,
            doNotWaitForReplication: doNotWaitForReplication = false,
            doNotWaitForNewlyAddedRemovals: doNotWaitForNewlyAddedRemovals = false,
            doNotWaitForPrimaryOnlyServices: doNotWaitForPrimaryOnlyServices = false,
            allNodesAuthorizedToRunRSGetStatus: allNodesAuthorizedToRunRSGetStatus = true,
        } = {},
    ) {
        let startTime = new Date(); // Measure the execution time of this function.
        this._initiateWithAnyNodeAsPrimary(cfg, initCmd, {
            doNotWaitForStableRecoveryTimestamp: doNotWaitForStableRecoveryTimestamp,
            doNotWaitForReplication: doNotWaitForReplication,
            doNotWaitForNewlyAddedRemovals: doNotWaitForNewlyAddedRemovals,
            doNotWaitForPrimaryOnlyServices: doNotWaitForPrimaryOnlyServices,
        });

        // stepUp() calls awaitReplication() which requires all nodes to be authorized to run
        // replSetGetStatus.
        if (!allNodesAuthorizedToRunRSGetStatus) {
            return;
        }

        // Most of the time node 0 will already be primary so we can skip the step-up.
        let primary = this.getPrimary();
        if (this.getNodeId(this.nodes[0]) == this.getNodeId(primary)) {
            jsTest.log.info(
                "ReplSetTest initiateWithNodeZeroAsPrimary skipping step-up because node 0 is " + "already primary",
            );
            asCluster(this, primary, () => {
                if (!doNotWaitForPrimaryOnlyServices) {
                    this.waitForStepUpWrites(primary);
                }
            });
        } else {
            asCluster(this, this.nodes, () => {
                const newPrimary = this.nodes[0];
                this.stepUp(newPrimary, {doNotWaitForPrimaryOnlyServices: doNotWaitForPrimaryOnlyServices});
                if (!doNotWaitForPrimaryOnlyServices) {
                    this.waitForStepUpWrites(newPrimary);
                }
            });
        }

        jsTest.log.info(
            "ReplSetTest initiateWithNodeZeroAsPrimary took " +
                (new Date() - startTime) +
                "ms for " +
                this.nodes.length +
                " nodes.",
        );
    }

    _addHighElectionTimeoutIfNotSet(config) {
        config = config || this.getReplSetConfig();
        config.settings = config.settings || {};
        config.settings["electionTimeoutMillis"] =
            config.settings["electionTimeoutMillis"] || ReplSetTest.kForeverMillis;
        return config;
    }

    /**
     * Initializes the replica set with `replSetInitiate`, setting a high election timeout unless
     * 'initiateWithDefaultElectionTimeout' is true. It requests the first node to step up as
     * primary. However, if 'allNodesAuthorizedToRunRSGetStatus' is set to false, any node can
     * become the primary.
     */
    initiate(
        cfg,
        initCmd,
        {
            doNotWaitForStableRecoveryTimestamp: doNotWaitForStableRecoveryTimestamp = false,
            doNotWaitForReplication: doNotWaitForReplication = false,
            doNotWaitForNewlyAddedRemovals: doNotWaitForNewlyAddedRemovals = false,
            doNotWaitForPrimaryOnlyServices: doNotWaitForPrimaryOnlyServices = false,
            initiateWithDefaultElectionTimeout: initiateWithDefaultElectionTimeout = false,
            allNodesAuthorizedToRunRSGetStatus: allNodesAuthorizedToRunRSGetStatus = true,
        } = {},
    ) {
        if (!initiateWithDefaultElectionTimeout) {
            cfg = this._addHighElectionTimeoutIfNotSet(cfg);
        }

        return this._initiateWithNodeZeroAsPrimary(cfg, initCmd, {
            doNotWaitForStableRecoveryTimestamp: doNotWaitForStableRecoveryTimestamp,
            doNotWaitForReplication: doNotWaitForReplication,
            doNotWaitForNewlyAddedRemovals: doNotWaitForNewlyAddedRemovals,
            doNotWaitForPrimaryOnlyServices: doNotWaitForPrimaryOnlyServices,
            allNodesAuthorizedToRunRSGetStatus: allNodesAuthorizedToRunRSGetStatus,
        });
    }

    /**
     * Runs replSetInitiate on the first node of the replica set.
     *
     * TODO (SERVER-109841): Replsetinitiate is currently a no-op command for disagg. Determine the
     * next steps for this function if additional functionality is to be incorporated.
     */
    initiateForDisagg(cfg, initCmd) {
        const startTime = new Date(); // Measure the execution time of this function.

        // Blocks until there is a primary. We use a faster retry interval here since we expect the
        // primary to be ready very soon. We also turn the failpoint off once we have a primary.
        this.getPrimary(this.kDefaultTimeoutMS, 25 /* retryIntervalMS */);

        jsTest.log(
            "ReplSetTest initiateForDisagg took " +
                (new Date() - startTime) +
                "ms for " +
                this.nodes.length +
                " nodes.",
        );
    }

    /**
     * Steps up 'node' as primary and by default it waits for the stepped up node to become a
     * writable primary and waits for all nodes to reach the same optime before sending the
     * replSetStepUp command to 'node'.
     *
     * Calls awaitReplication() which requires all connections in 'nodes' to be authenticated.
     * This stepUp() assumes that there is no network partition in the replica set.
     */
    stepUp(
        node,
        {
            awaitReplicationBeforeStepUp: awaitReplicationBeforeStepUp = true,
            awaitWritablePrimary: awaitWritablePrimary = true,
            doNotWaitForPrimaryOnlyServices = false,
        } = {},
    ) {
        jsTest.log("ReplSetTest stepUp: Stepping up " + node.host);

        if (awaitReplicationBeforeStepUp) {
            if (!doNotWaitForPrimaryOnlyServices) {
                this.waitForStepUpWrites();
            }
            this.awaitReplication();
        }

        assert.soonNoExcept(() => {
            const res = node.adminCommand({replSetStepUp: 1});
            // This error is possible if we are running mongoDB binary < 3.4 as
            // part of multi-version upgrade test. So, for those older branches,
            // simply wait for the requested node to get elected as primary due
            // to election timeout.
            if (!res.ok && res.code === ErrorCodes.CommandNotFound) {
                jsTest.log(
                    "replSetStepUp command not supported on node " +
                        node.host +
                        " ; so wait for the requested node to get elected due to election timeout.",
                );
                if (this.getPrimary() === node) {
                    return true;
                }
            }
            assert.commandWorked(res);

            // Since assert.soon() timeout is 10 minutes (default), setting
            // awaitNodesAgreeOnPrimary() timeout as 1 minute to allow retry of replSetStepUp
            // command on failure of the replica set to agree on the primary.
            // We should not run hangAnalyzer when awaitNodesAgreeOnPrimary() timeout, otherwise the
            // mongo processes will be killed and we cannot retry.
            const timeout = 60 * 1000;
            this.awaitNodesAgreeOnPrimary(timeout, this.nodes, node, false /*runHangAnalyzerOnTimeout*/);

            if (!awaitWritablePrimary) {
                return true;
            }

            // getPrimary() guarantees that there will be only one writable primary for a replica
            // set.
            const newPrimary = this.getPrimary();
            if (newPrimary.host === node.host) {
                return true;
            }

            jsTest.log(node.host + " is not primary after stepUp command, " + newPrimary.host + " is the primary");
            return false;
        }, "Timed out while waiting for stepUp to succeed on node in port: " + node.port);

        jsTest.log("ReplSetTest stepUp: Finished stepping up " + node.host);
        return node;
    }

    /**
     * Wait for writes which may happen when nodes are stepped up.  This currently includes
     * primary-only service writes and writes from the query analysis writer, the latter being
     * a replica-set-aware service for which there is no generic way to wait.
     */
    waitForStepUpWrites(primary) {
        primary = primary || this.getPrimary();
        this.waitForPrimaryOnlyServices(primary);
        this.waitForQueryAnalysisWriterSetup(primary);
    }

    /**
     * Waits for primary only services to finish the rebuilding stage after a primary is elected.
     * This is useful for tests that are expecting particular write timestamps since some primary
     * only services can do background writes (e.g. build indexes) during rebuilding stage that
     * could advance the last write timestamp.
     */
    waitForPrimaryOnlyServices(primary) {
        jsTest.log("Waiting for primary only services to finish rebuilding");
        primary = primary || this.getPrimary();

        assert.soonNoExcept(function () {
            const res = assert.commandWorked(primary.adminCommand({serverStatus: 1, repl: 1}));
            // 'PrimaryOnlyServices' does not exist prior to v5.0, using empty
            // object to skip waiting in case of multiversion tests.
            const services = res.repl.primaryOnlyServices || {};
            return Object.keys(services).every((s) => {
                return services[s].state === undefined || services[s].state === "running";
            });
        }, "Timed out waiting for primary only services to finish rebuilding");
    }

    /**
     * If query sampling is supported, waits for the query analysis writer to finish setting up
     * after a primary is elected. This is useful for tests that expect particular write timestamps
     * since the query analysis writer setup involves building indexes for the config.sampledQueries
     * and config.sampledQueriesDiff collections.
     */
    waitForQueryAnalysisWriterSetup(primary) {
        primary = primary || this.getPrimary();

        const serverStatusRes = assert.commandWorked(primary.adminCommand({serverStatus: 1}));
        if (!serverStatusRes.hasOwnProperty("queryAnalyzers")) {
            // Query sampling is not supported on this replica set. That is, either it uses binaries
            // released before query sampling was introduced or it uses binaries where query
            // sampling is guarded by a feature flag and the feature flag is not enabled.
            return;
        }

        const getParamsRes = primary.adminCommand({getParameter: 1, multitenancySupport: 1});
        if (!getParamsRes.ok || getParamsRes["multitenancySupport"]) {
            // Query sampling is not supported on a multi-tenant replica set.
            return;
        }

        jsTest.log("Waiting for query analysis writer to finish setting up");

        assert.soonNoExcept(function () {
            const sampledQueriesIndexes = primary.getCollection("config.sampledQueries").getIndexes();
            const sampledQueriesDiffIndexes = primary.getCollection("config.sampledQueriesDiff").getIndexes();
            // There should be two indexes: _id index and TTL index.
            return sampledQueriesIndexes.length == 2 && sampledQueriesDiffIndexes.length == 2;
        }, "Timed out waiting for query analysis writer to finish setting up");
    }

    /**
     * Gets the current replica set config from the specified node index. If no nodeId is specified,
     * uses the primary node.
     */
    getReplSetConfigFromNode(nodeId) {
        if (nodeId == undefined) {
            // Use 90 seconds timeout for finding a primary
            return _replSetGetConfig(this.getPrimary(90 * 1000));
        }

        if (!isNumber(nodeId)) {
            throw Error(nodeId + " is not a number");
        }

        return _replSetGetConfig(this.nodes[nodeId]);
    }

    reInitiate() {
        let config = this.getReplSetConfigFromNode();
        let newConfig = this.getReplSetConfig();
        // Only reset members.
        config.members = newConfig.members;
        config.version += 1;

        this._setDefaultConfigOptions(config);

        // Set a maxTimeMS so reconfig fails if it times out.
        assert.adminCommandWorkedAllowingNetworkError(this.getPrimary(), {
            replSetReconfig: config,
            maxTimeMS: this.timeoutMS,
        });
    }

    /**
     * Blocks until all nodes in the replica set have the same config version as the primary.
     **/
    awaitNodesAgreeOnConfigVersion(timeout) {
        timeout = timeout || this.timeoutMS;

        assert.soonNoExcept(
            () => {
                let primaryVersion = this.getPrimary().getDB("admin")._helloOrLegacyHello().setVersion;

                for (let i = 0; i < this.nodes.length; i++) {
                    let version = this.nodes[i].getDB("admin")._helloOrLegacyHello().setVersion;
                    assert.eq(
                        version,
                        primaryVersion,
                        "waiting for secondary node " +
                            this.nodes[i].host +
                            " with config version of " +
                            version +
                            " to match the version of the primary " +
                            primaryVersion,
                    );
                }

                return true;
            },
            "Awaiting nodes to agree on config version",
            timeout,
        );
    }

    /**
     * Waits for the last oplog entry on the primary to be visible in the committed snapshot view
     * of the oplog on *all* secondaries. When majority read concern is disabled, there is no
     * committed snapshot view, so this function waits for the knowledge of the majority commit
     * point on each node to advance to the optime of the last oplog entry on the primary.
     * Returns last oplog entry.
     */
    awaitLastOpCommitted(timeout, members) {
        let rst = this;
        let primary = rst.getPrimary();
        let primaryOpTime = _getLastOpTime(this, primary);

        let membersToCheck;
        if (members !== undefined) {
            jsTest.log.info("Waiting for op to be committed on " + members.map((s) => s.host), {opTime: primaryOpTime});

            membersToCheck = members;
        } else {
            jsTest.log.info("Waiting for op to be committed on all secondaries", {opTime: primaryOpTime});

            membersToCheck = rst.nodes;
        }

        assert.soonNoExcept(
            function () {
                for (let i = 0; i < membersToCheck.length; i++) {
                    var node = membersToCheck[i];
                    // Continue if we're connected to an arbiter
                    const res = asCluster(rst, node, () =>
                        assert.commandWorked(node.adminCommand({replSetGetStatus: 1})),
                    );

                    if (res.myState == ReplSetTest.State.ARBITER) {
                        continue;
                    }
                    let rcmOpTime = rst.getReadConcernMajorityOpTime(node);
                    if (friendlyEqual(rcmOpTime, {ts: Timestamp(0, 0), t: NumberLong(0)})) {
                        return false;
                    }
                    if (globalThis.rs.compareOpTimes(rcmOpTime, primaryOpTime) < 0) {
                        return false;
                    }
                }

                return true;
            },
            "Op with OpTime " + tojson(primaryOpTime) + " failed to be committed on all secondaries",
            timeout,
        );

        jsTest.log.info("Op successfully committed on all secondaries", {opTime: primaryOpTime});
        return primaryOpTime;
    }

    /**
     * This function performs some writes and then waits for all nodes in this replica set to
     * establish a stable recovery timestamp. The writes are necessary to prompt storage engines to
     * quickly establish stable recovery timestamps.
     *
     * A stable recovery timestamp ensures recoverable rollback is possible, as well as startup
     * recovery without re-initial syncing in the case of durable storage engines. By waiting for
     * all nodes to report having a stable recovery timestamp, we ensure a degree of stability in
     * our tests to run as expected.
     */
    awaitLastStableRecoveryTimestamp(retryIntervalMS) {
        let rst = this;
        let primary = rst.getPrimary();
        let id = tojson(rst.nodeList());
        retryIntervalMS = retryIntervalMS || 200;

        // All nodes must be in primary/secondary state prior to this point. Perform a majority
        // write to ensure there is a committed operation on the set. The commit point will
        // propagate to all members and trigger a stable checkpoint on all persisted storage engines
        // nodes.
        function advanceCommitPoint(rst, primary) {
            // Shadow 'db' so that we can call the function on the primary without a separate shell
            // when x509 auth is not needed.
            let db = primary.getDB("admin");
            const appendOplogNoteFn = function () {
                assert.commandWorked(
                    db.adminCommand({
                        "appendOplogNote": 1,
                        "data": {"awaitLastStableRecoveryTimestamp": 1},
                        // We use the global kDefaultTimeoutMS value since this func is passed to a new
                        // shell without context.
                        // TODO(SERVER-14017): Remove subshell use
                        "writeConcern": {"w": "majority", "wtimeout": ReplSetTest.kDefaultTimeoutMS},
                    }),
                );
            };

            runFnWithAuthOnPrimary(rst, appendOplogNoteFn, "AwaitLastStableRecoveryTimestamp");
        }

        jsTest.log.info("AwaitLastStableRecoveryTimestamp: Beginning for " + id);

        let replSetStatus = asCluster(rst, primary, () =>
            assert.commandWorked(primary.adminCommand({replSetGetStatus: 1})),
        );
        if (replSetStatus["configsvr"]) {
            // Performing dummy replicated writes against a configsvr is hard, especially if auth
            // is also enabled.
            return;
        }

        rst.awaitNodesAgreeOnPrimary();
        primary = rst.getPrimary();

        jsTest.log.info("AwaitLastStableRecoveryTimestamp: ensuring the commit point advances for " + id);
        advanceCommitPoint(this, primary);

        jsTest.log.info("AwaitLastStableRecoveryTimestamp: Waiting for stable recovery timestamps for " + id);

        assert.soonNoExcept(
            function () {
                for (let node of rst.nodes) {
                    // The `lastStableRecoveryTimestamp` field contains a stable timestamp
                    // guaranteed to exist on storage engine recovery to stable timestamp.
                    let res = asCluster(rst, node, () =>
                        assert.commandWorked(node.adminCommand({replSetGetStatus: 1})),
                    );

                    // Continue if we're connected to an arbiter.
                    if (res.myState === ReplSetTest.State.ARBITER) {
                        continue;
                    }

                    // A missing `lastStableRecoveryTimestamp` field indicates that the storage
                    // engine does not support `recover to a stable timestamp`.
                    //
                    // A null `lastStableRecoveryTimestamp` indicates that the storage engine
                    // supports "recover to a stable timestamp", but does not have a stable recovery
                    // timestamp yet.
                    if (
                        res.hasOwnProperty("lastStableRecoveryTimestamp") &&
                        res.lastStableRecoveryTimestamp.getTime() === 0
                    ) {
                        jsTest.log.info(
                            "AwaitLastStableRecoveryTimestamp: " +
                                node.host +
                                " does not have a stable recovery timestamp yet.",
                        );
                        return false;
                    }
                }

                return true;
            },
            "Not all members have a stable recovery timestamp",
            this.timeoutMS,
            retryIntervalMS,
        );

        jsTest.log.info(
            "AwaitLastStableRecoveryTimestamp: A stable recovery timestamp has successfully " + "established on " + id,
        );
    }

    // Wait until the optime of the specified type reaches the primary or the targetNode's last
    // applied optime if provided. Blocks on all secondary nodes or just 'secondaries', if
    // specified. The timeout will reset if any of the secondaries makes progress.
    awaitReplication(timeout, secondaryOpTimeType, secondaries, retryIntervalMS, targetNode) {
        if (secondaries !== undefined && secondaries !== this._secondaries) {
            jsTest.log.info("ReplSetTest awaitReplication: going to check only " + secondaries.map((s) => s.host));
        }

        if (targetNode !== undefined) {
            jsTest.log.info(
                `ReplSetTest awaitReplication: wait against targetNode ${targetNode.host} instead of primary.`,
            );
        }

        timeout = timeout || this.timeoutMS;
        retryIntervalMS = retryIntervalMS || 200;

        secondaryOpTimeType = secondaryOpTimeType || ReplSetTest.OpTimeType.LAST_APPLIED;

        let targetLatestOpTime;

        // Blocking call, which will wait for the last optime written on the target to be available
        let awaitLastOpTimeWrittenFn = function (rst) {
            let target = targetNode || rst.getPrimary();
            assert.soonNoExcept(
                function () {
                    try {
                        targetLatestOpTime = _getLastOpTime(rst, target);
                    } catch (e) {
                        jsTest.log.info("ReplSetTest caught exception", {error: e});
                        return false;
                    }

                    return true;
                },
                "awaiting oplog query",
                timeout,
            );
        };
        awaitLastOpTimeWrittenFn(this);

        // get the latest config version from target (with a few retries in case of error)
        let targetConfigVersion;
        let targetName;
        let target;
        let num_attempts = 3;

        assert.retryNoExcept(
            () => {
                target = targetNode || this.getPrimary();
                targetConfigVersion = asCluster(this, target, () =>
                    this.getReplSetConfigFromNode(this.getNodeId(target)),
                ).version;
                targetName = target.host;
                return true;
            },
            "ReplSetTest awaitReplication: couldnt get repl set config.",
            num_attempts,
            1000,
        );

        jsTest.log.info("ReplSetTest awaitReplication: starting: for target, " + targetName, {
            opTime: targetLatestOpTime,
        });

        let nodesCaughtUp = false;
        let secondariesToCheck = secondaries || this._secondaries;
        let nodeProgress = Array(secondariesToCheck.length);

        const Progress = Object.freeze({
            Skip: "Skip",
            CaughtUp: "CaughtUp",
            InProgress: "InProgress",
            Stuck: "Stuck",
            ConfigMismatch: "ConfigMismatch",
        });

        function checkProgressSingleNode(rst, index, secondaryCount) {
            let secondary = secondariesToCheck[index];
            let secondaryName = secondary.host;

            // TODO(SERVER-113063): Remove this skip.
            const shouldSkipConfigVersionCheck =
                typeof TestData !== "undefined" && TestData.skipAwaitReplicationConfigVersionCheck;
            if (!shouldSkipConfigVersionCheck) {
                let secondaryConfigVersion = asCluster(
                    rst,
                    secondary,
                    () =>
                        secondary.getDB("local")["system.replset"].find().readConcern("local").limit(1).next().version,
                );

                if (targetConfigVersion != secondaryConfigVersion) {
                    jsTest.log.info(
                        "ReplSetTest awaitReplication: secondary #" +
                            secondaryCount +
                            ", " +
                            secondaryName +
                            ", has config version #" +
                            secondaryConfigVersion +
                            ", but expected config version #" +
                            targetConfigVersion,
                    );

                    if (secondaryConfigVersion > targetConfigVersion) {
                        target = targetNode || rst.getPrimary();
                        targetConfigVersion = target
                            .getDB("local")
                            ["system.replset"].find()
                            .readConcern("local")
                            .limit(1)
                            .next().version;
                        targetName = target.host;

                        jsTest.log.info("ReplSetTest awaitReplication: for target, " + targetName, {
                            opTime: targetLatestOpTime,
                        });
                    }

                    return Progress.ConfigMismatch;
                }
            }

            // Skip this node if we're connected to an arbiter
            let res = asCluster(rst, secondary, () =>
                assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1})),
            );
            if (res.myState == ReplSetTest.State.ARBITER) {
                return Progress.Skip;
            }

            jsTest.log.info(
                "ReplSetTest awaitReplication: checking secondary #" + secondaryCount + ": " + secondaryName,
            );

            secondary.getDB("admin").getMongo().setSecondaryOk();

            let secondaryOpTime;
            if (secondaryOpTimeType == ReplSetTest.OpTimeType.LAST_DURABLE) {
                secondaryOpTime = _getDurableOpTime(rst, secondary);
            } else {
                secondaryOpTime = _getLastOpTime(rst, secondary);
            }

            // If the node doesn't have a valid opTime, it likely hasn't received any writes from
            // the primary yet.
            if (!globalThis.rs.isValidOpTime(secondaryOpTime)) {
                jsTest.log.info(
                    "ReplSetTest awaitReplication: optime for secondary #" +
                        secondaryCount +
                        ", " +
                        secondaryName +
                        ", is NOT valid.",
                    {opTime: secondaryOpTime},
                );
                return Progress.Stuck;
            }

            // See if the node made progress. We count it as progress even if the node's last optime
            // went backwards because that means the node is in rollback.
            let madeProgress =
                nodeProgress[index] && globalThis.rs.compareOpTimes(nodeProgress[index], secondaryOpTime) != 0;
            nodeProgress[index] = secondaryOpTime;

            if (globalThis.rs.compareOpTimes(targetLatestOpTime, secondaryOpTime) < 0) {
                targetLatestOpTime = _getLastOpTime(rst, target);
                jsTest.log.info(
                    "ReplSetTest awaitReplication: optime for " +
                        secondaryName +
                        " is newer, resetting latest target optime. Also resetting awaitReplication timeout.",
                    {resetOpTime: targetLatestOpTime},
                );
                return Progress.InProgress;
            }

            if (!friendlyEqual(targetLatestOpTime, secondaryOpTime)) {
                jsTest.log.info(
                    "ReplSetTest awaitReplication: optime for secondary #" +
                        secondaryCount +
                        ", " +
                        secondaryName +
                        ", is different than latest optime",
                    {secondaryOpTime, targetLatestOpTime},
                );
                jsTest.log.info(
                    "ReplSetTest awaitReplication: secondary #" +
                        secondaryCount +
                        ", " +
                        secondaryName +
                        ", is NOT synced",
                );

                // Reset the timeout if a node makes progress, but isn't caught up yet.
                if (madeProgress) {
                    jsTest.log.info(
                        "ReplSetTest awaitReplication: secondary #" +
                            secondaryCount +
                            ", " +
                            secondaryName +
                            ", has made progress. Resetting awaitReplication timeout",
                    );
                    return Progress.InProgress;
                }
                return Progress.Stuck;
            }

            jsTest.log.info(
                "ReplSetTest awaitReplication: secondary #" + secondaryCount + ", " + secondaryName + ", is synced",
            );
            return Progress.CaughtUp;
        }

        // We will reset the timeout if a nodes makes progress, but still isn't caught up yet.
        while (!nodesCaughtUp) {
            assert.soonNoExcept(
                () => {
                    try {
                        jsTest.log.info(
                            "ReplSetTest awaitReplication: checking secondaries against latest target optime",
                            {targetLatestOpTime},
                        );
                        let secondaryCount = 0;

                        for (let i = 0; i < secondariesToCheck.length; i++) {
                            const action = checkProgressSingleNode(this, i, secondaryCount);

                            switch (action) {
                                case Progress.CaughtUp:
                                    // We only need to increment the secondaryCount if this node is
                                    // caught up.
                                    secondaryCount++;
                                    continue;
                                case Progress.Skip:
                                    // Don't increment secondaryCount because this node is an arbiter.
                                    continue;
                                case Progress.InProgress:
                                    return true;
                                case Progress.Stuck:
                                case Progress.ConfigMismatch:
                                    return false;
                                default:
                                    throw Error("invalid action: " + tojson(action));
                            }
                        }

                        jsTest.log.info(
                            "ReplSetTest awaitReplication: finished: all " + secondaryCount + " secondaries synced",
                            {opTime: targetLatestOpTime},
                        );
                        nodesCaughtUp = true;
                        return true;
                    } catch (e) {
                        jsTest.log.info("ReplSetTest awaitReplication: caught exception", {error: e});

                        // We might have a new primary now
                        awaitLastOpTimeWrittenFn(this);

                        jsTest.log.info("ReplSetTest awaitReplication: resetting: for target " + target, {
                            opTime: targetLatestOpTime,
                        });

                        return false;
                    }
                },
                "awaitReplication timed out",
                timeout,
                retryIntervalMS,
            );
        }
    }

    getHashesUsingSessions(
        sessions,
        dbName,
        {readAtClusterTime, skipTempCollections = false} = {
            skipTempCollections: false,
        },
    ) {
        return sessions.map((session) => {
            const commandObj = {dbHash: 1};
            const db = session.getDatabase(dbName);
            // Use snapshot read concern for dbhash.
            if (readAtClusterTime !== undefined) {
                commandObj.readConcern = {level: "snapshot", atClusterTime: readAtClusterTime};
            }
            if (skipTempCollections) {
                commandObj.skipTempCollections = 1;
            }
            // If we are running in a multiversion suite, preserve old behavior of checking capped
            // collections in _id order instead of natural order. The
            // 'useIndexScanForCappedCollections' option for dbHash should be ignored in older
            // binaries.
            if (
                typeof TestData !== "undefined" &&
                (TestData.useRandomBinVersionsWithinReplicaSet ||
                    TestData.mongosBinVersion ||
                    TestData.multiversionBinVersion ||
                    TestData.mixedBinVersions)
            ) {
                commandObj.useIndexScanForCappedCollections = 1;
            }

            return assert.commandWorked(db.runCommand(commandObj));
        });
    }

    // Gets the dbhash for the current primary and for all secondaries (or the members of
    // 'secondaries', if specified).
    getHashes(dbName, secondaries, skipTempCollections) {
        assert.neq(dbName, "local", 'Cannot run getHashes() on the "local" database');

        // _determineLiveSecondaries() repopulates both 'self._secondaries' and 'self._primary'. If
        // we're passed an explicit set of secondaries we don't want to do that.
        secondaries = secondaries || _determineLiveSecondaries(this);

        const sessions = [
            this._primary,
            ...secondaries.filter((conn) => {
                return !conn.getDB("admin")._helloOrLegacyHello().arbiterOnly;
            }),
        ].map((conn) => conn.getDB("test").getSession());

        const hashes = this.getHashesUsingSessions(sessions, dbName, {skipTempCollections});
        return {primary: hashes[0], secondaries: hashes.slice(1)};
    }

    findOplog(conn, query, limit) {
        return conn.getDB("local").getCollection(kOplogName).find(query).sort({$natural: -1}).limit(limit);
    }

    dumpOplog(conn, query = {}, limit = 10) {
        let log =
            "Dumping the latest " +
            limit +
            " documents that match " +
            tojson(query) +
            " from the oplog " +
            kOplogName +
            " of " +
            conn.host;
        let entries = [];
        let cursor = this.findOplog(conn, query, limit);
        cursor.forEach(function (entry) {
            log = log + "\n" + tojsononeline(entry);
            entries.push(entry);
        });
        jsTestLog(log);
        return entries;
    }

    // Call the provided checkerFunction, after the replica set has been write locked.
    checkReplicaSet(checkerFunction, secondaries, ...checkerFunctionArgs) {
        assert.eq(typeof checkerFunction, "function", "Expected checkerFunction parameter to be a function");

        assert(secondaries, "must pass list of live nodes to checkReplicaSet");

        // Call getPrimary to populate rst with information about the nodes.
        let primary = this.getPrimary();
        assert(primary, "calling getPrimary() failed");

        // Ensure that the current primary isn't in the secondaries list from a stale
        // determineLiveSecondaries call. Otherwise we will mistakenly freeze the current primary.
        const primIndex = secondaries.indexOf(primary);
        if (primIndex > -1) {
            secondaries.splice(primIndex, 1);
        }
        checkerFunctionArgs.push(secondaries);

        // Prevent an election, which could start, then hang due to the fsyncLock.
        jsTestLog(`Freezing nodes: [${secondaries.map((n) => n.host)}]`);
        secondaries.forEach((secondary) => this.freeze(secondary));

        // Await primary in case freeze() had to step down a node that was unexpectedly primary.
        this.getPrimary();

        // Lock the primary to prevent writes in the background while we are getting the
        // dbhashes of the replica set members. It's not important if the storage engine fails
        // to perform its fsync operation. The only requirement is that writes are locked out.
        assert.commandWorked(
            primary.adminCommand({fsync: 1, lock: 1, allowFsyncFailure: true}),
            "failed to lock the primary",
        );

        function postApplyCheckerFunction() {
            // Unfreeze secondaries and unlock primary.
            try {
                assert.commandWorked(primary.adminCommand({fsyncUnlock: 1}));
            } catch (e) {
                jsTest.log.info("Continuing after fsyncUnlock error", {error: e});
            }

            secondaries.forEach((secondary) => {
                try {
                    assert.commandWorked(secondary.adminCommand({replSetFreeze: 0}));
                } catch (e) {
                    jsTest.log.info("Continuing after replSetFreeze error", {error: e});
                }
            });
        }

        let activeException = false;
        try {
            this.awaitReplication(null, null, secondaries);
            checkerFunction.apply(this, checkerFunctionArgs);
        } catch (e) {
            activeException = true;
            throw e;
        } finally {
            if (activeException) {
                try {
                    postApplyCheckerFunction();
                } catch (e) {
                    // Print the postApplyCheckerFunction error, propagate the original.
                    jsTest.log.info({error: e});
                }
            } else {
                postApplyCheckerFunction();
            }
        }
    }

    // Check the replicated data hashes for all live nodes in the set.
    checkReplicatedDataHashes(msgPrefix = "checkReplicatedDataHashes", excludedDBs = [], ignoreUUIDs = false) {
        // Return items that are in either Array `a` or `b` but not both. Note that this will
        // not work with arrays containing NaN. Array.indexOf(NaN) will always return -1.

        let collectionPrinted = new Set();

        function checkDBHashesForReplSet(rst, dbDenylist = [], msgPrefix, ignoreUUIDs, secondaries) {
            // We don't expect the local database to match because some of its
            // collections are not replicated.
            dbDenylist.push("local");
            secondaries = secondaries || rst._secondaries;

            let success = true;
            let hasDumpedOplog = false;

            // Use '_primary' instead of getPrimary() to avoid the detection of a new primary.
            // '_primary' must have been populated.
            const primary = rst._primary;

            let combinedDBs = new Map();
            primary.getDBs().databases.map((db) => {
                const key = `${db.tenantId}_${db.name}`;
                const obj = {"name": db.name, "tenant": db.tenantId};
                combinedDBs.set(key, obj);
            });

            const replSetConfig = rst.getReplSetConfigFromNode();

            jsTest.log.info("checkDBHashesForReplSet waiting for secondaries to be ready", {secondaries});
            this.awaitSecondaryNodes(rst.timeoutMS, secondaries);

            jsTest.log.info("checkDBHashesForReplSet checking data hashes against primary: " + primary.host);

            secondaries.forEach((node) => {
                // Arbiters have no replicated data.
                if (isNodeArbiter(node)) {
                    jsTest.log.info("checkDBHashesForReplSet skipping data of arbiter: " + node.host);
                    return;
                }
                jsTest.log.info("checkDBHashesForReplSet going to check data hashes on secondary: " + node.host);
                node.getDBs().databases.forEach((db) => {
                    const key = `${db.tenantId}_${db.name}`;
                    const obj = {"name": db.name, "tenant": db.tenantId};
                    combinedDBs.set(key, obj);
                });
            });

            const expectPrefix = typeof TestData !== "undefined" && TestData.multitenancyExpectPrefix ? true : false;

            for (const [key, db] of combinedDBs) {
                const tenant = db.tenant;
                const dbName = expectPrefix && tenant ? tenant + "_" + db.name : db.name;

                if (Array.contains(dbDenylist, dbName)) {
                    continue;
                }

                const token = db.tenant ? _createTenantToken({tenant, expectPrefix}) : undefined;
                try {
                    primary._setSecurityToken(token);
                    secondaries.forEach((node) => node._setSecurityToken(token));

                    const dbHashes = rst.getHashes(dbName, secondaries);
                    const primaryDBHash = dbHashes.primary;
                    const primaryCollections = Object.keys(primaryDBHash.collections);
                    assert.commandWorked(primaryDBHash);

                    // Filter only collections that were retrieved by the dbhash.
                    // listCollections may include non-replicated collections like
                    // system.profile.
                    const primaryCollInfos = new CollInfos(primary, "primary", dbName);
                    primaryCollInfos.filter(primaryCollections);

                    dbHashes.secondaries.forEach((secondaryDBHash) => {
                        assert.commandWorked(secondaryDBHash);

                        const secondary = secondaryDBHash._mongo;
                        const secondaryCollections = Object.keys(secondaryDBHash.collections);
                        // Check that collection information is consistent on the primary and
                        // secondaries.
                        const secondaryCollInfos = new CollInfos(secondary, "secondary", dbName);
                        secondaryCollInfos.filter(secondaryCollections);

                        const hasSecondaryIndexes =
                            replSetConfig.members[rst.getNodeId(secondary)].buildIndexes !== false;

                        jsTest.log.info(
                            `checking db hash between primary: ${primary.host}, and secondary: ${secondary.host}`,
                        );
                        success =
                            DataConsistencyChecker.checkDBHash(
                                primaryDBHash,
                                primaryCollInfos,
                                secondaryDBHash,
                                secondaryCollInfos,
                                msgPrefix,
                                ignoreUUIDs,
                                hasSecondaryIndexes,
                                collectionPrinted,
                            ) && success;

                        if (!success) {
                            if (!hasDumpedOplog) {
                                jsTest.log.info("checkDBHashesForReplSet dumping oplogs from all nodes");
                                this.dumpOplog(primary, {}, 100);
                                rst.getSecondaries().forEach((secondary) => this.dumpOplog(secondary, {}, 100));
                                hasDumpedOplog = true;
                            }
                        }
                    });
                } finally {
                    primary._setSecurityToken(undefined);
                    secondaries.forEach((node) => node._setSecurityToken(undefined));
                }
            }

            assert(success, "dbhash mismatch between primary and secondary");
        }

        const liveSecondaries = _determineLiveSecondaries(this);
        this.checkReplicaSet(checkDBHashesForReplSet, liveSecondaries, this, excludedDBs, msgPrefix, ignoreUUIDs);
    }

    checkOplogs(msgPrefix) {
        let liveSecondaries = _determineLiveSecondaries(this);
        this.checkReplicaSet(checkOplogs, liveSecondaries, this, msgPrefix);
    }

    checkPreImageCollection(msgPrefix) {
        let liveSecondaries = _determineLiveSecondaries(this);
        this.checkReplicaSet(checkPreImageCollection, liveSecondaries, this, msgPrefix);
    }

    checkChangeCollection(msgPrefix) {
        let liveSecondaries = _determineLiveSecondaries(this);
        this.checkReplicaSet(checkChangeCollection, liveSecondaries, this, msgPrefix);
    }

    /**
     * Waits for an initial connection to a given node. Should only be called after the node's
     * process has already been started. Updates the corresponding entry in 'this.nodes' with the
     * newly established connection object.
     *
     * @private
     * @param {int} [n] the node id.
     * @param {boolean} [waitForHealth] If true, wait for the health indicator of the replica set
     *     node after waiting for a connection. Default: false.
     * @returns a new Mongo connection object to the node.
     */
    _waitForInitialConnection(n, waitForHealth) {
        jsTest.log.info("ReplSetTest waiting for an initial connection to node " + n);

        // If we are using a bridge, then we want to get at the underlying mongod node object.
        let node = this._useBridge ? this._unbridgedNodes[n] : this.nodes[n];
        let pid = node.pid;
        let port = node.port;
        let conn = MongoRunner.awaitConnection({pid, port});
        if (!conn) {
            throw new Error("Failed to connect to node " + n);
        }

        // Attach the original node properties to the connection object.
        Object.assign(conn, node);

        // Delete the session since it's linked to the other mongo object.
        delete conn._defaultSession;

        // Authenticate again since this is a new connection.
        if (jsTestOptions().keyFile || this.clusterAuthMode === "x509") {
            // The sslSpecial suite sets up cluster with x509 but the shell was not started with TLS
            // so we need to rely on the test to auth if needed.
            if (!(this.clusterAuthMode === "x509" && !conn.isTLS())) {
                jsTest.authenticate(conn);
            }
        }

        // Save the new connection object. If we are using a bridge, then we need to connect to it.
        if (this._useBridge) {
            this.nodes[n].connectToBridge();
            this.nodes[n].nodeId = n;
            this._unbridgedNodes[n] = conn;
        } else {
            this.nodes[n] = conn;
        }

        jsTest.log.info("ReplSetTest made initial connection to node", {node: this.nodes[n]});

        waitForHealth = waitForHealth || false;
        if (waitForHealth) {
            // Wait for node to start up.
            this._waitForIndicator(this.nodes[n], "health", Health.UP);
        }

        if (this._causalConsistency) {
            this.nodes[n].setCausalConsistency(true);
        }
        return this.nodes[n];
    }

    /**
     * Starts up a server.  Options are saved by default for subsequent starts.
     *
     * @param {int|conn|[int|conn]} n array or single server number (0, 1, 2, ...) or conn
     * @param {object} [options]
     * @param {boolean} [options.remember] Reapplies the saved options from a prior start.
     * @param {boolean} [options.noRemember] Ignores the current properties.
     * @param {boolean} [options.appendOptions] Appends the current options to those remembered.
     * @param {boolean} [options.startClean] Clears the data directory before starting.
     * @param {boolean} [restart=false] If false, the data directory will be cleared before the
     *     server starts.
     * @param {boolean} [waitForHealth=false] If true, wait for the health indicator of the replica
     *     set node after waiting for a connection.
     */
    start(n, options, restart, waitForHealth) {
        n = resolveToNodeId(this, n);
        jsTest.log.info("ReplSetTest n is : " + n);

        let defaults = {
            useHostName: this.useHostName,
            oplogSize: this.oplogSize,
            keyFile: this.keyFile,
            port: this._useBridge ? this._unbridgedPorts[n] : this.ports[n],
            dbpath: "$set-$node",
        };
        if (this._maintenancePorts?.[n] > 0) {
            defaults.maintenancePort = this._maintenancePorts[n];
        }
        if (jsTestOptions().shellGRPC) {
            defaults.grpcPort = this.grpcPorts[n];
        }

        if (this.useAutoBootstrapProcedure) {
            if (n == 0) {
                // No --replSet for the first node.
            } else {
                defaults.replSet = this.name;
            }
        } else {
            defaults.replSet = this.useSeedList ? this.getURL() : this.name;
        }

        const nodeOptions = this.nodeOptions["n" + n];
        const hasBinVersion = (options && options.binVersion) || (nodeOptions && nodeOptions.binVersion);
        if (hasBinVersion && jsTest.options().useRandomBinVersionsWithinReplicaSet) {
            throw new Error("Can only specify one of binVersion and useRandomBinVersionsWithinReplicaSet, not both.");
        }

        // Note : this replaces the binVersion of the shared startSet() options the first time
        // through, so the full set is guaranteed to have different versions if size > 1.  If using
        // start() independently, independent version choices will be made
        //
        if (options && options.binVersion) {
            options.binVersion = MongoRunner.versionIterator(options.binVersion);
        }

        // Always set log format
        if (options && options.logFormat) {
            options.logFormat = jsTest.options().logFormat;
        }

        // If restarting a node, use its existing options as the defaults unless remember is false.
        let baseOptions;
        if ((options && options.restart) || restart) {
            if (options && options.remember === false) {
                baseOptions = defaults;
            } else {
                baseOptions = this._useBridge ? this._unbridgedNodes[n].fullOptions : this.nodes[n].fullOptions;
            }
        } else {
            baseOptions = defaults;
        }
        baseOptions = Object.merge(baseOptions, nodeOptions);
        options = Object.merge(baseOptions, options);
        if (options.hasOwnProperty("rsConfig")) {
            this.nodeOptions["n" + n] = Object.merge(this.nodeOptions["n" + n], {rsConfig: options.rsConfig});
        }
        delete options.rsConfig;

        if (jsTest.options().useRandomBinVersionsWithinReplicaSet) {
            if (this.isConfigServer) {
                // Our documented upgrade/downgrade paths for a sharded cluster lets us assume that
                // config server nodes will always be fully upgraded before the shard nodes.
                options.binVersion = "latest";
            } else {
                const rand = Random.rand();
                options.binVersion = rand < 0.5 ? "latest" : jsTest.options().useRandomBinVersionsWithinReplicaSet;
            }
            jsTest.log.info("Randomly assigned binary version: " + options.binVersion + " to node: " + n);
        }

        options.restart = options.restart || restart;

        let pathOpts = {node: n, set: this.name};
        options.pathOpts = Object.merge(options.pathOpts || {}, pathOpts);

        // Turn off periodic noop writes for replica sets by default.
        options.setParameter = options.setParameter || {};
        if (typeof options.setParameter === "string") {
            let eqIdx = options.setParameter.indexOf("=");
            if (eqIdx != -1) {
                let param = options.setParameter.substring(0, eqIdx);
                let value = options.setParameter.substring(eqIdx + 1);
                options.setParameter = {};
                options.setParameter[param] = value;
            }
        }
        options.setParameter.writePeriodicNoops = options.setParameter.writePeriodicNoops || false;

        // We raise the number of initial sync connect attempts for tests that disallow chaining.
        // Disabling chaining can cause sync source selection to take longer so we must increase
        // the number of connection attempts.
        options.setParameter.numInitialSyncConnectAttempts = options.setParameter.numInitialSyncConnectAttempts || 60;

        // The default time for stepdown and quiesce mode in response to SIGTERM is 15 seconds.
        // Reduce this to 100ms for faster shutdown.
        options.setParameter.shutdownTimeoutMillisForSignaledShutdown =
            options.setParameter.shutdownTimeoutMillisForSignaledShutdown || 100;

        if (jsTestOptions().enableTestCommands) {
            // This parameter is enabled to allow the default write concern to change while
            // initiating a ReplSetTest. This is due to our testing optimization to initiate
            // with a single node, and reconfig the full membership set in.
            // We need to recalculate the DWC after each reconfig until the full set is included.
            options.setParameter.enableDefaultWriteConcernUpdatesForInitiate = true;
        }

        if (
            baseOptions.hasOwnProperty("setParameter") &&
            baseOptions.setParameter.hasOwnProperty("featureFlagTransitionToCatalogShard") &&
            baseOptions.setParameter.featureFlagTransitionToCatalogShard
        ) {
            options.setParameter.featureFlagTransitionToCatalogShard = true;
        }

        // Disable a check in reconfig that will prevent certain configs with arbiters from
        // spinning up. We will re-enable this check after the replica set has finished initiating.
        if (jsTestOptions().enableTestCommands) {
            options.setParameter.enableReconfigRollbackCommittedWritesCheck = false;
        }

        if (jsTestOptions().performTimeseriesCompressionIntermediateDataIntegrityCheckOnInsert) {
            options.setParameter.performTimeseriesCompressionIntermediateDataIntegrityCheckOnInsert = true;
        }

        if (this.useAutoBootstrapProcedure) {
            options.setParameter.featureFlagAllMongodsAreSharded = true;
        }

        if (typeof TestData !== "undefined" && TestData.replicaSetEndpointIncompatible) {
            options.setParameter.featureFlagReplicaSetEndpoint = false;
        }

        const olderThan73 =
            MongoRunner.compareBinVersions(
                MongoRunner.getBinVersionFor("7.3"),
                MongoRunner.getBinVersionFor(options.binVersion),
            ) === 1;
        if (olderThan73) {
            delete options.setParameter.featureFlagClusteredConfigTransactions;
            delete options.setParameter.featureFlagReplicaSetEndpoint;
        }

        const olderThan81 =
            MongoRunner.compareBinVersions(
                MongoRunner.getBinVersionFor(options.binVersion),
                MongoRunner.getBinVersionFor("8.1"),
            ) === -1;
        if (olderThan81) {
            delete options.setParameter.performTimeseriesCompressionIntermediateDataIntegrityCheckOnInsert;
        }

        if (tojson(options) != tojson({})) jsTest.log.info({options});

        jsTest.log.info("ReplSetTest " + (restart ? "(Re)" : "") + "Starting....");

        if (this._useBridge && (restart === undefined || !restart)) {
            // We leave the mongobridge process running when the mongod process is restarted so we
            // don't need to start a new one.
            let bridgeOptions = Object.merge(this._bridgeOptions, options.bridgeOptions || {});
            bridgeOptions = Object.merge(bridgeOptions, {
                hostName: this.host,
                port: this.ports[n],
                // The mongod processes identify themselves to mongobridge as host:port, where the
                // host is the actual hostname of the machine and not localhost.
                dest: getHostName() + ":" + this._unbridgedPorts[n],
            });

            if (jsTestOptions().networkMessageCompressors) {
                bridgeOptions["networkMessageCompressors"] = jsTestOptions().networkMessageCompressors;
            }

            this.nodes[n] = new MongoBridge(bridgeOptions);
        }

        // Save this property since it may be deleted inside 'runMongod'.
        let waitForConnect = options.waitForConnect;

        // Never wait for a connection inside runMongod. We will do so below if needed.
        options.waitForConnect = false;
        let conn = MongoRunner.runMongod(options);
        if (!conn) {
            throw new Error("Failed to start node " + n);
        }

        // Make sure to call _addPath, otherwise folders won't be cleaned.
        this._addPath(conn.dbpath);

        // We don't want to persist 'waitForConnect' across node restarts.
        delete conn.fullOptions.waitForConnect;

        // Save the node object in the appropriate location.
        if (this._useBridge) {
            this._unbridgedNodes[n] = conn;
        } else {
            this.nodes[n] = conn;
            this.nodes[n].nodeId = n;
        }

        // Clean up after noReplSet to ensure it doesn't effect future restarts.
        if (options.noReplSet) {
            this.nodes[n].fullOptions.replSet = defaults.replSet;
            delete this.nodes[n].fullOptions.noReplSet;
        }

        // Wait for a connection to the node if necessary.
        if (waitForConnect === false) {
            jsTest.log.info("ReplSetTest start skip waiting for a connection to node " + n);
            return this.nodes[n];
        }

        const connection = this._waitForInitialConnection(n, waitForHealth);

        if (n == 0 && this.useAutoBootstrapProcedure && !this._hasAcquiredAutoGeneratedName) {
            const helloReply = connection.getDB("admin")._helloOrLegacyHello();
            jsTest.log.info(
                "ReplSetTest start using auto generated replSet name " +
                    helloReply.setName +
                    " instead of " +
                    this.name,
            );
            this.name = helloReply.setName;
            this._hasAcquiredAutoGeneratedName = true;
        }

        return connection;
    }

    /**
     * Restarts a db without clearing the data directory by default, and using the node(s)'s
     * original startup options by default.
     *
     * When using this method with mongobridge, be aware that mongobridge may not do a good
     * job of detecting that a node was restarted. For example, when mongobridge is being used
     * between some Node A and Node B, on restarting Node B mongobridge will not aggressively
     * close its connection with Node A, leading Node A to think the connection with Node B is
     * still healthy.
     *
     * In order not to use the original startup options, use stop() (or stopSet()) followed by
     * start() (or startSet()) without passing restart: true as part of the options.
     *
     * @param {int|conn|[int|conn]} n array or single server number (0, 1, 2, ...) or conn
     * @param {Object} [options]
     * @param {boolean} [options.startClean] Forces clearing the data directory.
     * @param {Object} [options.auth] Object that contains the auth details for admin credentials.
     *     Should contain the fields 'user' and 'pwd'.
     */
    restart(n, options, signal, wait) {
        n = resolveToNodeId(this, n);

        // Can specify wait as third parameter, if using default signal
        if (signal == true || signal == false) {
            wait = signal;
            signal = undefined;
        }

        this.stop(n, signal, options, {forRestart: true});

        let started = this.start(n, options, true, wait);

        // We should not attempt to reauthenticate the connection if we did not wait for it
        // to be reestablished in the first place.
        const skipWaitForConnection = options && options.waitForConnect === false;
        if (jsTestOptions().keyFile && !skipWaitForConnection) {
            if (started.length) {
                // if n was an array of conns, start will return an array of connections
                for (let i = 0; i < started.length; i++) {
                    assert(jsTest.authenticate(started[i]), "Failed authentication during restart");
                }
            } else {
                assert(jsTest.authenticate(started), "Failed authentication during restart");
            }
        }
        return started;
    }

    /**
     * Step down and freeze a particular node.
     *
     * @param node A single node you wish to freeze
     */
    freeze(node) {
        node = resolveToConnection(this, node);

        assert.soon(
            () => {
                try {
                    // Ensure node is authenticated.
                    asCluster(this, node, () => {
                        // Ensure node is not primary. Ignore errors, probably means it's already
                        // secondary.
                        node.adminCommand({replSetStepDown: ReplSetTest.kForeverSecs, force: true});
                        // Prevent node from running election. Fails if it already started an election.
                        assert.commandWorked(node.adminCommand({replSetFreeze: ReplSetTest.kForeverSecs}));
                    });
                    return true;
                } catch (e) {
                    if (
                        isNetworkError(e) ||
                        e.code === ErrorCodes.NotSecondary ||
                        e.code === ErrorCodes.NotYetInitialized
                    ) {
                        jsTestLog(`Failed to freeze node ${node.host}: ${e}`);
                        return false;
                    }

                    throw e;
                }
            },
            `Failed to run replSetFreeze cmd on ${node.host}`,
            this.timeoutMS,
        );
    }

    /**
     * Unfreeze a particular node or nodes.
     *
     * @param node is a single node or list of nodes, by id or conn
     */
    unfreeze(node) {
        node = resolveToConnection(this, node);

        // Ensure node is authenticated.
        asCluster(this, node, () => assert.commandWorked(node.adminCommand({replSetFreeze: 0})));
    }

    stopPrimary(signal, opts) {
        let primary = this.getPrimary();
        let primary_id = this.getNodeId(primary);
        return this.stop(primary_id, signal, opts);
    }

    /**
     * Stops a particular node or nodes, specified by conn or id. If we expect the node to exit with
     * a nonzero exit code, call this function and pass in allowedExitCode as a field of opts.
     *
     * If _useBridge=true, then the mongobridge process(es) corresponding to the node(s) are also
     * terminated unless forRestart=true. The mongobridge process(es) are left running across
     * restarts to ensure their configuration remains intact.
     *
     * @param {number|Mongo} n the index or connection object of the replica set member to stop.
     * @param {number} signal the signal number to use for killing
     * @param {Object} opts @see MongoRunner.stopMongod
     * @param {Object} [extraOptions={}]
     * @param {boolean} [extraOptions.forRestart=false] indicates whether stop() is being called
     * with the intent to call start() with restart=true for the same node(s) n.
     * @param {boolean} [extraOptions.waitPid=true] if true, we will wait for the process to
     * terminate after stopping it.
     */
    stop(n, signal, opts, {forRestart: forRestart = false, waitpid: waitPid = true} = {}) {
        n = resolveToNodeId(this, n);

        // Can specify wait as second parameter, if using default signal
        if (signal == true || signal == false) {
            signal = undefined;
        }

        let conn = this._useBridge ? this._unbridgedNodes[n] : this.nodes[n];

        jsTest.log.info(
            "ReplSetTest stop *** Shutting down mongod in port " +
                conn.port +
                ", wait for process termination: " +
                waitPid +
                " ***",
        );
        let ret = MongoRunner.stopMongod(conn, signal, opts, waitPid);

        // We only expect the process to have terminated if we actually called 'waitpid'.
        if (waitPid) {
            jsTest.log.info(
                "ReplSetTest stop *** Mongod in port " + conn.port + " shutdown with code (" + ret + ") ***",
            );
        }

        if (this._useBridge && !forRestart) {
            // We leave the mongobridge process running when the mongod process is being restarted.
            const bridge = this.nodes[n];
            jsTest.log.info("ReplSetTest stop *** Shutting down mongobridge on port " + bridge.port + " ***");
            const exitCode = bridge.stop(); // calls MongoBridge#stop()
            jsTest.log.info(
                "ReplSetTest stop *** mongobridge on port " + bridge.port + " exited with code (" + exitCode + ") ***",
            );
        }

        return ret;
    }

    /**
     * Performs collection validation on all nodes in the given 'ports' array in parallel.
     *
     * @private
     * @param {int[]} ports the array of mongo ports to run validation on
     */
    _validateNodes(ports) {
        // Perform collection validation on each node in parallel.
        let validators = [];
        for (let i = 0; i < ports.length; i++) {
            const validator = new Thread(async function (port) {
                const {CommandSequenceWithRetries} = await import("jstests/libs/command_sequence_with_retries.js");
                const {validateCollections} = await import("jstests/hooks/validate_collections.js");
                await import("jstests/libs/override_methods/validate_collections_on_shutdown.js");
                MongoRunner.validateCollectionsCallback(port, {CommandSequenceWithRetries, validateCollections});
            }, ports[i]);
            validators.push(validator);
            validators[i].start();
        }
        // Wait for all validators to finish.
        for (let i = 0; i < ports.length; i++) {
            validators[i].join();
        }
    }

    isReplicaSetEndpointActive() {
        _callHello(this);

        for (let node of this._liveNodes) {
            const helloRes = node.getDB("admin")._helloOrLegacyHello();
            if (!helloRes.configsvr && !this.useAutoBootstrapProcedure) {
                return false;
            }

            let shardDocs;
            try {
                shardDocs = asCluster(this, node, () => node.getCollection("config.shards").find().toArray());
            } catch (e) {
                if (e.code == ErrorCodes.NotPrimaryOrSecondary) {
                    // This node has been removed from the replica set.
                    continue;
                }
                throw e;
            }
            if (shardDocs.length != 1) {
                return false;
            }
            if (shardDocs[0]._id != "config") {
                return false;
            }
            return asCluster(this, node, () => {
                const serverStatusRes = assert.commandWorked(node.adminCommand({serverStatus: 1}));
                const olderThan73 =
                    MongoRunner.compareBinVersions(
                        MongoRunner.getBinVersionFor("7.3"),
                        MongoRunner.getBinVersionFor(serverStatusRes.version),
                    ) === 1;
                if (olderThan73) {
                    return false;
                }
                const getParameterRes = assert.commandWorked(
                    node.adminCommand({getParameter: 1, featureFlagReplicaSetEndpoint: 1}),
                );
                return getParameterRes.featureFlagReplicaSetEndpoint.value;
            });
        }
        return false;
    }

    /**
     * Kill all members of this replica set. When calling this function, we expect all live nodes to
     * exit cleanly. If we expect a node to exit with a nonzero exit code, use the stop function to
     * terminate that node before calling stopSet.
     *
     * @param {number} signal The signal number to use for killing the members
     * @param {boolean} forRestart will not cleanup data directory
     * @param {Object} opts @see MongoRunner.stopMongod
     */
    stopSet(signal, forRestart, opts = {}) {
        if (jsTestOptions().alwaysUseLogFiles) {
            if (opts.noCleanData === false) {
                throw new Error("Always using log files, but received conflicting option.");
            }

            opts.noCleanData = true;
        }

        const primary = _callHello(this);
        // TODO (SERVER-83433): Add back the test coverage for running db hash check and validation
        // on replica set that is fsync locked and has replica set endpoint enabled.
        if (
            (!opts.hasOwnProperty("skipCheckDBHashes") || !opts.hasOwnProperty("skipValidation")) &&
            primary &&
            this._liveNodes.length > 0 &&
            this.isReplicaSetEndpointActive()
        ) {
            opts = Object.assign({}, opts, {skipCheckDBHashes: true, skipValidation: true});
        }

        // Check to make sure data is the same on all nodes.
        const skipChecks = jsTest.options().skipCheckDBHashes || (opts && opts.skipCheckDBHashes);
        if (!skipChecks) {
            let startTime = new Date(); // Measure the execution time of consistency checks.
            jsTest.log.info("ReplSetTest stopSet going to run data consistency checks.");
            // To skip this check add TestData.skipCheckDBHashes = true or pass in {opts:
            // skipCheckDBHashes} Reasons to skip this test include:
            // - the primary goes down and none can be elected (so fsync lock/unlock commands fail)
            // - the replica set is in an unrecoverable inconsistent state. E.g. the replica set
            //   is partitioned.
            if (primary && this._liveNodes.length > 1) {
                // skip for sets with 1 live node
                // Auth only on live nodes because authutil.assertAuthenticate
                // refuses to log in live connections if some secondaries are down.
                jsTest.log.info("ReplSetTest stopSet checking oplogs.");
                asCluster(this, this._liveNodes, () => this.checkOplogs());
                jsTest.log.info("ReplSetTest stopSet checking preimages.");
                asCluster(this, this._liveNodes, () => this.checkPreImageCollection());
                jsTest.log.info("ReplSetTest stopSet checking change_collection(s).");
                asCluster(this, this._liveNodes, () => this.checkChangeCollection());
                jsTest.log.info("ReplSetTest stopSet checking replicated data hashes.");
                asCluster(this, this._liveNodes, () => this.checkReplicatedDataHashes());
            } else {
                jsTest.log.info(
                    "ReplSetTest stopSet skipped data consistency checks. Number of _liveNodes: " +
                        this._liveNodes.length +
                        ", _callHello response: " +
                        primary,
                );
            }
            jsTest.log.info(
                "ReplSetTest stopSet data consistency checks finished, took " +
                    (new Date() - startTime) +
                    "ms for " +
                    this.nodes.length +
                    " nodes.",
            );
        }

        let startTime = new Date(); // Measure the execution time of shutting down nodes.

        if (opts.skipValidation) {
            jsTest.log.info("ReplSetTest stopSet skipping validation before stopping nodes.");
        } else {
            jsTest.log.info("ReplSetTest stopSet validating all replica set nodes before stopping them.");
            this._validateNodes(this.ports);
        }

        // Stop all nodes without waiting for them to terminate. We can skip validation on shutdown
        // since we have already done it above (or validation was explicitly skipped).
        opts = Object.merge(opts, {skipValidation: true});
        for (let i = 0; i < this.ports.length; i++) {
            this.stop(i, signal, opts, {waitpid: false});
        }

        // Wait for all processes to terminate.
        for (let i = 0; i < this.ports.length; i++) {
            let conn = this._useBridge ? this._unbridgedNodes[i] : this.nodes[i];
            let port = parseInt(conn.name.split(":")[1]);
            jsTest.log.info("ReplSetTest stopSet waiting for mongo program on port " + port + " to stop.");
            let exitCode = waitMongoProgram(port);
            if (exitCode !== MongoRunner.EXIT_CLEAN && !opts.skipValidatingExitCode) {
                throw new Error(
                    "ReplSetTest stopSet mongo program on port " +
                        port +
                        " shut down unexpectedly with code " +
                        exitCode +
                        " when code " +
                        MongoRunner.EXIT_CLEAN +
                        " was expected.",
                );
            }
            jsTest.log.info("ReplSetTest stopSet mongo program on port " + port + " shut down with code " + exitCode);
        }

        jsTest.log.info(
            "ReplSetTest stopSet stopped all replica set nodes, took " +
                (new Date() - startTime) +
                "ms for " +
                this.ports.length +
                " nodes.",
        );

        if (forRestart) {
            jsTest.log.info("ReplSetTest stopSet returning since forRestart=true.");
            return;
        }

        if (!opts.noCleanData && this._alldbpaths) {
            jsTest.log.info("ReplSetTest stopSet deleting all dbpaths");
            for (let i = 0; i < this._alldbpaths.length; i++) {
                jsTest.log.info("ReplSetTest stopSet deleting dbpath: " + this._alldbpaths[i]);
                resetDbpath(this._alldbpaths[i]);
            }
            jsTest.log.info("ReplSetTest stopSet deleted all dbpaths");
        }

        _forgetReplSet(this.name);

        jsTest.log.info("ReplSetTest stopSet *** Shut down repl set - test worked ****");
    }

    /**
     * Returns whether or not this ReplSetTest uses mongobridge.
     */
    usesBridge() {
        return this._useBridge;
    }

    /**
     * Wait for a state indicator to go to a particular state or states.
     *
     * Note that this waits for the state as indicated by the primary node, if there is one. If not,
     * it will use the first live node.
     *
     * Cannot be used to wait for a secondary state alone. To wait for a secondary state, use the
     * function 'awaitSecondaryNodes' instead.
     *
     * @param node is a single node, by id or conn
     * @param state is a single state or list of states
     * @param timeout how long to wait for the state to be reached
     * @param reconnectNode indicates that we should reconnect to a node that stepped down
     */
    waitForState(node, state, timeout, reconnectNode) {
        assert(
            state != ReplSetTest.State.SECONDARY,
            "To wait for a secondary state, use the function 'awaitSecondaryNodes' instead.",
        );
        this._waitForIndicator(node, "state", state, timeout, reconnectNode);
    }

    /**
     * Waits until there is a primary node.
     */
    waitForPrimary(timeout) {
        let primary;
        assert.soonNoExcept(
            () => {
                return (primary = this.getPrimary());
            },
            "waiting for primary",
            timeout,
        );

        return primary;
    }

    /**
     * Returns after stable_timestamp has been advanced to at least Timestamp ts. The stable
     * timestamp is not exposed by replSetGetStatus, so we use the readConcernMajorityOpTime
     * instead, because it reflects the time of the current committed snapshot. This time is
     * determined before setting the stable timestamp, and both are done under the same mutex
     * acquisition, so readConcernMajorityOpTime >= ts indicates stable_timestamp >= ts. See
     * ReplicationCoordinatorImpl::_setStableTimestampForStorage for further
     * implementation details.
     * @param node The node to check for stable_timestamp.
     * @param ts The timestamp to compare the stable_timestamp to.
     * @param timeout How long to wait for state, defaults to global value.
     */
    waitForStableTimestampTobeAdvanced(node, ts, timeout = ReplSetTest.kDefaultTimeoutMS) {
        assert.soon(
            function () {
                jsTestLog("Waiting for stable_timestamp >= Timestamp " + ts.toStringIncomparable());
                const replSetStatus = assert.commandWorked(node.adminCommand({replSetGetStatus: 1}));
                const readConcernMajorityOpTime = replSetStatus.optimes.readConcernMajorityOpTime.ts;
                return timestampCmp(readConcernMajorityOpTime, ts) >= 0;
            },
            "Timed out waiting for stable_timestamp",
            timeout,
        );
    }

    /**
     * Returns after lastStableRecoveryTimestamp has been advanced to at least Timestamp ts. Note
     * that the checkpointer thread should be running for this function to return.
     * @param node The node to check for last checkpoint time.
     * @param ts The timestamp to compare the lastStableRecoveryTimestamp to.
     * @param timeout How long to wait for state, defaults to global value.
     */
    waitForCheckpoint(node, ts, timeout = ReplSetTest.kDefaultTimeoutMS) {
        this.waitForStableTimestampTobeAdvanced(node, ts, timeout);
        assert.soon(
            function () {
                jsTestLog("Waiting for checkpoint >= Timestamp " + ts.toStringIncomparable());
                const replSetStatus = assert.commandWorked(node.adminCommand({replSetGetStatus: 1}));
                const lastStableRecoveryTimestamp = replSetStatus.lastStableRecoveryTimestamp;
                return timestampCmp(lastStableRecoveryTimestamp, ts) >= 0;
            },
            "Timed out waiting for checkpoint",
            timeout,
        );
    }
}

/**
 *  Global default timeout (10 minutes).
 */
ReplSetTest.kDefaultTimeoutMS = 10 * 60 * 1000;
ReplSetTest.kDefaultRetries = 240;

/**
 *  Global default number that's effectively infinite.
 */
ReplSetTest.kForeverSecs = 24 * 60 * 60;
ReplSetTest.kForeverMillis = ReplSetTest.kForeverSecs * 1000;

/**
 * Set of states that the replica set can be in. Used for the wait functions.
 */
ReplSetTest.State = {
    PRIMARY: 1,
    SECONDARY: 2,
    RECOVERING: 3,
    // Note there is no state 4
    STARTUP_2: 5,
    UNKNOWN: 6,
    ARBITER: 7,
    DOWN: 8,
    ROLLBACK: 9,
    REMOVED: 10,
};

ReplSetTest.OpTimeType = {
    LAST_APPLIED: 1,
    LAST_DURABLE: 2,
};

/**
 * Constructor, which initializes the ReplSetTest object by starting new instances.
 */
function _constructStartNewInstances(rst, opts) {
    rst.name = opts.name || jsTest.name();
    jsTest.log.info("Starting new replica set " + rst.name);

    rst.useHostName = opts.useHostName == undefined ? true : opts.useHostName;
    rst.host = rst.useHostName ? opts.host || getHostName() : "localhost";
    rst.oplogSize = opts.oplogSize || 40;
    rst.useSeedList = opts.useSeedList || false;
    rst.keyFile = opts.keyFile;

    rst.clusterAuthMode = undefined;
    if (opts.clusterAuthMode) {
        rst.clusterAuthMode = opts.clusterAuthMode;
    }

    rst.protocolVersion = opts.protocolVersion;
    rst.waitForKeys = opts.waitForKeys;

    rst.seedRandomNumberGenerator = opts.hasOwnProperty("seedRandomNumberGenerator")
        ? opts.seedRandomNumberGenerator
        : true;
    rst.isConfigServer = opts.isConfigServer;

    rst._useBridge = opts.useBridge || false;
    if (rst._useBridge) {
        assert(
            !jsTestOptions().tlsMode,
            "useBridge cannot be true when using TLS. Add the requires_mongobridge tag to the test to ensure it will be skipped on variants that use TLS.",
        );
    }

    rst._bridgeOptions = opts.bridgeOptions || {};

    rst._useMaintenancePorts = opts.useMaintenancePorts ?? false;
    if (rst._useMaintenancePorts) {
        assert(!rst._useBridge, "useMaintenancePorts is not supported when using MongoBridge.");
    }

    rst._causalConsistency = opts.causallyConsistent || false;

    rst._configSettings = opts.settings || false;
    rst.useAutoBootstrapProcedure = opts.useAutoBootstrapProcedure || false;
    rst._hasAcquiredAutoGeneratedName = false;

    rst.nodeOptions = {};

    let numNodes;

    if (isObject(opts.nodes)) {
        let len = 0;
        for (var i in opts.nodes) {
            // opts.nodeOptions and opts.nodes[i] may contain nested objects that have
            // the same key, e.g. setParameter. So we need to recursively merge them.
            // Object.assign and Object.merge do not merge nested objects of the same key.
            let options = (rst.nodeOptions["n" + len] = _deepObjectMerge(opts.nodeOptions, opts.nodes[i]));
            if (i.startsWith("a")) {
                options.arbiter = true;
            }

            len++;
        }

        numNodes = len;
    } else if (Array.isArray(opts.nodes)) {
        for (var i = 0; i < opts.nodes.length; i++) {
            rst.nodeOptions["n" + i] = Object.merge(opts.nodeOptions, opts.nodes[i]);
        }

        numNodes = opts.nodes.length;
    } else {
        for (var i = 0; i < opts.nodes; i++) {
            rst.nodeOptions["n" + i] = opts.nodeOptions;
        }

        numNodes = opts.nodes;
    }

    for (let i = 0; i < numNodes; i++) {
        if (rst.nodeOptions["n" + i] !== undefined && rst.nodeOptions["n" + i].clusterAuthMode == "x509") {
            rst.clusterAuthMode = "x509";
        }
    }

    if (rst._useBridge) {
        let makeAllocatePortFn = (preallocatedPorts) => {
            let idxNextNodePort = 0;

            return function () {
                if (idxNextNodePort >= preallocatedPorts.length) {
                    throw new Error(
                        "Cannot use a replica set larger than " +
                            preallocatedPorts.length +
                            " members with useBridge=true",
                    );
                }

                const nextPort = preallocatedPorts[idxNextNodePort];
                ++idxNextNodePort;
                return nextPort;
            };
        };

        rst._allocatePortForBridge = makeAllocatePortFn(allocatePorts(MongoBridge.kBridgeOffset));
        rst._allocatePortForNode = makeAllocatePortFn(allocatePorts(MongoBridge.kBridgeOffset));
    } else {
        rst._allocatePortForBridge = function () {
            throw new Error("Using mongobridge isn't enabled for this replica set");
        };
        rst._allocatePortForNode = allocatePort;
    }

    rst.nodes = [];

    if (rst._useBridge) {
        rst.ports = Array.from({length: numNodes}, rst._allocatePortForBridge);
        rst._unbridgedPorts = Array.from({length: numNodes}, rst._allocatePortForNode);
        rst._unbridgedNodes = [];
    } else {
        rst.ports = opts.ports || Array.from({length: numNodes}, rst._allocatePortForNode);
        if (rst._useMaintenancePorts) {
            rst._maintenancePorts = Array.from({length: numNodes}, rst._allocatePortForNode);
        }
    }

    for (let i = 0; i < numNodes; i++) {
        const nodeOpts = rst.nodeOptions["n" + i];
        if (nodeOpts && nodeOpts.hasOwnProperty("port")) {
            if (rst._useBridge) {
                rst._unbridgedPorts[i] = nodeOpts.port;
            } else {
                rst.ports[i] = nodeOpts.port;
            }
        }
        if (nodeOpts?.hasOwnProperty("maintenancePort")) {
            if (!rst._maintenancePorts) {
                rst._maintenancePorts = Array(numNodes).fill(-1);
            }
            rst._maintenancePorts[i] = nodeOpts.maintenancePort;
        }
    }

    if (jsTestOptions().shellGRPC) {
        rst.grpcPorts = Array.from({length: numNodes}, rst._allocatePortForNode);
    }
}

function _newMongo(host) {
    return new Mongo(host, undefined, {gRPC: false});
}

/**
 * Constructor, which instantiates the ReplSetTest object from an existing set.
 */
function _constructFromExistingSeedNode(rst, seedNode) {
    const conn = _newMongo(seedNode);
    if (jsTest.options().keyFile) {
        rst.keyFile = jsTest.options().keyFile;
    }
    let conf = asCluster(rst, conn, () => _replSetGetConfig(conn));
    jsTest.log.info("Recreating replica set from config", {conf});

    let existingNodes = conf.members.map((member) => member.host);
    rst.ports = existingNodes.map((node) => node.split(":")[1]);
    rst.nodes = existingNodes.map((node) => {
        // Note: the seed node is required to be operational in order for the Mongo
        // shell to connect to it. In this code there is no fallback to other nodes.
        let conn = _newMongo(node);
        conn.name = conn.host;
        return conn;
    });
    rst.waitForKeys = false;
    rst.host = existingNodes[0].split(":")[0];
    rst.name = conf._id;
}

/**
 * Constructor, which instantiates the ReplSetTest object from existing nodes.
 */
function _constructFromExistingNodes(
    rst,
    {name, nodeHosts, nodeOptions, keyFile, host, waitForKeys, useAutoBootstrapProcedure, pidValue = undefined},
) {
    jsTest.log.info("Recreating replica set from existing nodes", {nodeHosts});

    rst.name = name;
    rst.ports = nodeHosts.map((node) => node.split(":")[1]);

    let i = 0;
    rst.nodes = nodeHosts.map((node) => {
        const conn = _newMongo(node);
        conn.name = conn.host;
        conn.port = node.split(":")[1];
        if (pidValue !== undefined && pidValue[i] !== undefined) {
            conn.pid = pidValue[i];
            i++;
        }
        return conn;
    });

    rst.host = host;
    rst.waitForKeys = waitForKeys;
    rst.keyFile = keyFile;
    rst.nodeOptions = nodeOptions;
    rst.useAutoBootstrapProcedure = useAutoBootstrapProcedure || false;
}

/**
 * Returns the config document reported from the specified connection.
 */
function _replSetGetConfig(conn) {
    return assert.commandWorked(conn.adminCommand({replSetGetConfig: 1})).config;
}

/**
 * Invokes the 'hello' command on each individual node and returns the current primary, or false
 * if none is found. Populates the following cached values:
 * '_primary': the current primary
 * '_secondaries': all nodes other than '_primary' (note this includes arbiters)
 * '_liveNodes': all currently reachable nodes
 */
function _callHello(rst) {
    rst._liveNodes = [];
    rst._primary = null;
    rst._secondaries = [];

    let twoPrimaries = false;
    let canAcceptWrites = false;
    // Ensure that only one node is in primary state.
    rst.nodes.forEach(function (node) {
        try {
            node.setSecondaryOk();
            let n = node.getDB("admin")._helloOrLegacyHello();
            rst._liveNodes.push(node);
            // We verify that the node has a valid config by checking if n.me exists. Then, we
            // check to see if the node is in primary state.
            if (n.me && n.me == n.primary) {
                if (rst._primary) {
                    twoPrimaries = true;
                } else {
                    rst._primary = node;
                    canAcceptWrites = n.isWritablePrimary || n.ismaster;
                }
            } else {
                rst._secondaries.push(node);
            }
        } catch (err) {
            jsTest.log.info("ReplSetTest Could not call hello/ismaster on node " + node, {error: err});
            rst._secondaries.push(node);
        }
    });
    if (twoPrimaries || !rst._primary || !canAcceptWrites) {
        return false;
    }

    return rst._primary;
}

/**
 * Attempt to connect to all nodes and returns a list of secondaries in which the connection was
 * successful.
 */
function _determineLiveSecondaries(rst) {
    _callHello(rst);
    return rst._secondaries.filter(function (n) {
        return rst._liveNodes.indexOf(n) !== -1;
    });
}

/**
 * For all unauthenticated connections passed in, authenticates them with the '__system' user.
 * If a connection is already authenticated, we will skip authentication for that connection and
 * assume that it already has the correct privileges. It is up to the caller of this function to
 * ensure that the connection is appropriately authenticated.
 */
function asCluster(rst, conn, fn, keyFileParam = undefined) {
    let connArray = conn;
    if (conn.length == null) connArray = [conn];

    let rst_keyfile = null;
    if (rst !== undefined && rst !== null) {
        rst_keyfile = rst.keyFile;
    }

    const unauthenticatedConns = connArray.filter((connection) => {
        const connStatus = connection.adminCommand({connectionStatus: 1, showPrivileges: true});
        const connIsAuthenticated = connStatus.authInfo.authenticatedUsers.length > 0;
        return !connIsAuthenticated;
    });

    const connOptions = connArray[0].fullOptions || {};
    const authMode = connOptions.clusterAuthMode || connArray[0].clusterAuthMode || jsTest.options().clusterAuthMode;

    keyFileParam = keyFileParam || connOptions.keyFile || rst_keyfile;
    let needsAuth =
        (keyFileParam || authMode === "x509" || authMode === "sendX509" || authMode === "sendKeyFile") &&
        unauthenticatedConns.length > 0;

    // There are few cases where we do not auth
    // 1. When transitioning to auth
    // 2. When cluster is running in x509 but shell was not started with TLS (i.e. sslSpecial
    // suite)
    if (needsAuth && (connOptions.transitionToAuth !== undefined || (authMode === "x509" && !connArray[0].isTLS()))) {
        needsAuth = false;
    }

    if (needsAuth) {
        return authutil.asCluster(unauthenticatedConns, keyFileParam, fn);
    } else {
        return fn();
    }
}

/**
 * Returns 'true' if the "conn" has been configured to run without journaling enabled.
 */
function _isRunningWithoutJournaling(rst, conn) {
    let result = asCluster(rst, conn, function () {
        // Persistent storage engines (WT) can only run with journal enabled.
        let serverStatus = assert.commandWorked(conn.adminCommand({serverStatus: 1}));
        if (serverStatus.storageEngine.hasOwnProperty("persistent")) {
            if (serverStatus.storageEngine.persistent) {
                return false;
            }
        }
        return true;
    });
    return result;
}

/**
 * Helper functions for setting/clearing a failpoint.
 */
function setFailPoint(node, failpoint, data = {}) {
    jsTest.log.info("Setting fail point " + failpoint);
    assert.commandWorked(node.adminCommand({configureFailPoint: failpoint, mode: "alwaysOn", data: data}));
}

function clearFailPoint(node, failpoint) {
    jsTest.log.info("Clearing fail point " + failpoint);
    assert.commandWorked(node.adminCommand({configureFailPoint: failpoint, mode: "off"}));
}

/**
 * Returns true if the OpTime is empty, else false.
 *
 * Empty OpTime Formats:
 *   PV0: Timestamp(0,0)
 *   PV1: {ts: Timestamp(0,0), t: NumberLong(-1)}
 */
function _isEmptyOpTime(opTime) {
    if (!opTime.hasOwnProperty("ts") || !opTime.hasOwnProperty("t")) {
        return opTime.getTime() == 0 && opTime.getInc() == 0;
    }
    return opTime.ts.getTime() == 0 && opTime.ts.getInc() == 0 && opTime.t == -1;
}

/**
 * Returns the OpTime for the specified host by issuing replSetGetStatus.
 */
function _getLastOpTime(rst, conn) {
    let replSetStatus = asCluster(rst, conn, () =>
        assert.commandWorked(conn.getDB("admin").runCommand({replSetGetStatus: 1})),
    );
    let connStatus = replSetStatus.members.filter((m) => m.self)[0];
    let opTime = connStatus.optime;
    if (_isEmptyOpTime(opTime)) {
        throw new Error("last OpTime is empty -- connection: " + conn);
    }
    return opTime;
}

/**
 * Returns the last durable OpTime for the host if running with journaling.
 * Returns the last applied OpTime otherwise.
 */
function _getDurableOpTime(rst, conn) {
    let replSetStatus = asCluster(rst, conn, () =>
        assert.commandWorked(conn.getDB("admin").runCommand({replSetGetStatus: 1})),
    );

    let opTimeType = "durableOpTime";
    if (_isRunningWithoutJournaling(rst, conn)) {
        opTimeType = "appliedOpTime";
    }
    let opTime = replSetStatus.optimes[opTimeType];
    if (_isEmptyOpTime(opTime)) {
        throw new Error("last durable OpTime is empty -- connection: " + conn);
    }
    return opTime;
}

/*
 * Returns true if the node can be elected primary of a replica set.
 */
function _isElectable(node) {
    return !node.arbiterOnly && (node.priority === undefined || node.priority != 0);
}

function isNodeArbiter(node) {
    return node.getDB("admin")._helloOrLegacyHello().arbiterOnly;
}

function replSetCommandWithRetry(primary, cmd) {
    jsTest.log.info("Running command with retry", {cmd});
    const cmdName = Object.keys(cmd)[0];
    const errorMsg = `${cmdName} during initiate failed`;
    assert.retry(
        () => {
            const result = assert.commandWorkedOrFailedWithCode(
                primary.runCommand(cmd),
                [
                    ErrorCodes.NodeNotFound,
                    ErrorCodes.NewReplicaSetConfigurationIncompatible,
                    ErrorCodes.InterruptedDueToReplStateChange,
                    ErrorCodes.ConfigurationInProgress,
                    ErrorCodes.CurrentConfigNotCommittedYet,
                    ErrorCodes.NotWritablePrimary,
                ],
                errorMsg,
            );
            return result.ok;
        },
        errorMsg,
        3,
        5 * 1000,
    );
}

// TODO(SERVER-14017): Remove this extra sub-shell in favor of a cleaner authentication
// solution.
function runFnWithAuthOnPrimary(rst, fn, fnName) {
    const primary = rst.getPrimary();
    const primaryId = "n" + rst.getNodeId(primary);
    const primaryOptions = rst.nodeOptions[primaryId] || {};
    const options = Object.keys(primaryOptions).length !== 0 || !rst.startOptions ? primaryOptions : rst.startOptions;
    const authMode = options.clusterAuthMode;
    if (authMode === "x509") {
        jsTest.log.info(fnName + ": authenticating on separate shell with x509 for " + rst.name);
        const caFile = options.sslCAFile ? options.sslCAFile : options.tlsCAFile;
        const keyFile = options.sslPEMKeyFile ? options.sslPEMKeyFile : options.tlsCertificateKeyFile;
        const subShellArgs = [
            "mongo",
            "--ssl",
            "--sslCAFile=" + caFile,
            "--sslPEMKeyFile=" + keyFile,
            "--sslAllowInvalidHostnames",
            "--authenticationDatabase=$external",
            "--authenticationMechanism=MONGODB-X509",
            primary.host,
            "--eval",
            `import {ReplSetTest} from "jstests/libs/replsettest.js"; (${fn.toString()})();`,
        ];

        const retVal = _runMongoProgram(...subShellArgs);
        assert.eq(retVal, 0, "mongo shell did not succeed with exit code 0");
    } else {
        jsTest.log.info(fnName + ": authenticating with authMode '" + authMode + "' for " + rst.name);
        asCluster(rst, primary, fn, primaryOptions.keyFile);
    }
}

const ReverseReader = function (mongo, coll, query) {
    this.kCappedPositionLostSentinel = Object.create(null);

    this._safelyPerformCursorOperation = function (name, operation, onCappedPositionLost) {
        if (!this.cursor) {
            throw new Error("ReverseReader is not open!");
        }

        if (this._cursorExhausted) {
            return onCappedPositionLost;
        }

        try {
            return operation(this.cursor);
        } catch (err) {
            jsTest.log.info("Error: " + name + " threw '" + err.message + "' on " + this.mongo.host);
            // Occasionally, the capped collection will get truncated while we are iterating
            // over it. Since we are iterating over the collection in reverse, getting a
            // truncated item means we've reached the end of the list, so return false.
            if (err.code === ErrorCodes.CappedPositionLost) {
                this.cursor.close();
                this._cursorExhausted = true;
                return onCappedPositionLost;
            }

            throw err;
        }
    };

    this.next = function () {
        return this._safelyPerformCursorOperation(
            "next",
            function (cursor) {
                return cursor.next();
            },
            this.kCappedPositionLostSentinel,
        );
    };

    this.hasNext = function () {
        return this._safelyPerformCursorOperation(
            "hasNext",
            function (cursor) {
                return cursor.hasNext();
            },
            false,
        );
    };

    this.query = function () {
        // Set the cursor to read backwards, from last to first. We also set the cursor not
        // to time out since it may take a while to process each batch and a test may have
        // changed "cursorTimeoutMillis" to a short time period.
        // TODO SERVER-75496 remove the batchSize once the the following issue is fixed: The
        // find{...} will always run with apiStrict:false, however getMore may run with
        // apiStrict: true on specific suites. Use a big batch size to prevent getMore from
        // running.
        this._cursorExhausted = false;
        this.cursor = coll.find(query).sort({$natural: -1}).noCursorTimeout().readConcern("local").batchSize(200);
    };

    this.getFirstDoc = function () {
        return coll.find(query).sort({$natural: 1}).readConcern("local").limit(-1).next();
    };

    this.cursor = null;
    this._cursorExhausted = true;
    this.mongo = mongo;
};

/**
 * Check oplogs on all nodes, by reading from the last time. Since the oplog is a capped
 * collection, each node may not contain the same number of entries and stop if the cursor
 * is exhausted on any node being checked.
 *
 * `secondaries` must be the last argument since in checkReplicaSet we explicitly append the live
 * secondaries to the end of the parameter list after ensuring that the current primary is excluded.
 */
function checkOplogs(rst, msgPrefix = "checkOplogs", secondaries) {
    secondaries = secondaries || rst._secondaries;

    function assertOplogEntriesEq(oplogEntry0, oplogEntry1, reader0, reader1, prevOplogEntry) {
        if (!bsonBinaryEqual(oplogEntry0, oplogEntry1)) {
            const query = prevOplogEntry ? {ts: {$lte: prevOplogEntry.ts}} : {};
            rst.nodes.forEach((node) => rst.dumpOplog(node, query, 100));
            const log =
                msgPrefix +
                ", non-matching oplog entries for the following nodes: \n" +
                reader0.mongo.host +
                ": " +
                tojsononeline(oplogEntry0) +
                "\n" +
                reader1.mongo.host +
                ": " +
                tojsononeline(oplogEntry1);
            assert(false, log);
        }
    }

    jsTest.log.info("checkOplogs starting oplog checks.");
    jsTest.log.info("checkOplogs waiting for secondaries to be ready.");
    rst.awaitSecondaryNodes(rst.timeoutMS, secondaries);
    if (secondaries.length >= 1) {
        let readers = [];
        let smallestTS = new Timestamp(Math.pow(2, 32) - 1, Math.pow(2, 32) - 1);
        const nodes = rst.nodes;
        let firstReaderIndex;
        for (let i = 0; i < nodes.length; i++) {
            const node = nodes[i];

            if (rst._primary !== node && !secondaries.includes(node)) {
                jsTest.log.info("checkOplogs skipping oplog of node: " + node.host);
                continue;
            }

            // Arbiters have no documents in the oplog.
            if (isNodeArbiter(node)) {
                jsTestLog("checkOplogs skipping oplog of arbiter: " + node.host);
                continue;
            }

            jsTest.log.info("checkOplogs going to check oplog of node: " + node.host);
            readers[i] = new ReverseReader(node, node.getDB("local")[kOplogName], {ts: {$gte: new Timestamp()}});
            const currTS = readers[i].getFirstDoc().ts;
            // Find the reader which has the smallestTS. This reader should have the most
            // number of documents in the oplog.
            if (timestampCmp(currTS, smallestTS) < 0) {
                smallestTS = currTS;
                firstReaderIndex = i;
            }
            // Start all oplogReaders at their last document.
            readers[i].query();
        }

        // Read from the reader which has the most oplog entries.
        // Note, we read the oplog backwards from last to first.
        const firstReader = readers[firstReaderIndex];
        let prevOplogEntry;
        assert(firstReader.hasNext(), "oplog is empty while checkOplogs is called");
        // Track the number of bytes we are reading as we check the oplog. We use this to avoid
        // out-of-memory issues by calling to garbage collect whenever the memory footprint is
        // large.
        let bytesSinceGC = 0;
        while (firstReader.hasNext()) {
            const oplogEntry = firstReader.next();
            bytesSinceGC += Object.bsonsize(oplogEntry);
            if (oplogEntry === firstReader.kCappedPositionLostSentinel) {
                // When using legacy OP_QUERY/OP_GET_MORE reads against mongos, it is
                // possible for hasNext() to return true but for next() to throw an exception.
                break;
            }

            for (let i = 0; i < nodes.length; i++) {
                // Skip reading from this reader if the index is the same as firstReader or
                // the cursor is exhausted.
                if (i === firstReaderIndex || !(readers[i] && readers[i].hasNext())) {
                    continue;
                }

                const otherOplogEntry = readers[i].next();
                bytesSinceGC += Object.bsonsize(otherOplogEntry);
                if (otherOplogEntry && otherOplogEntry !== readers[i].kCappedPositionLostSentinel) {
                    assertOplogEntriesEq.call(
                        this,
                        oplogEntry,
                        otherOplogEntry,
                        firstReader,
                        readers[i],
                        prevOplogEntry,
                    );
                }
            }
            // Garbage collect every 10MB.
            if (bytesSinceGC > 10 * 1024 * 1024) {
                gc();
                bytesSinceGC = 0;
            }
            prevOplogEntry = oplogEntry;
        }
    }
    jsTest.log.info("checkOplogs oplog checks complete.");
}

function getPreImageReaders(msgPrefix, rst, secondaries, nsUUID) {
    const readers = [];
    const nodes = rst.nodes;
    for (let i = 0; i < nodes.length; i++) {
        const node = nodes[i];

        if (rst._primary !== node && !secondaries.includes(node)) {
            jsTest.log.info(
                `${msgPrefix} -- skipping preimages of node as it's not in our list of ` + `secondaries: ${node.host}`,
            );
            continue;
        }

        // Arbiters have no documents in the oplog and thus don't have preimages
        // content.
        if (isNodeArbiter(node)) {
            jsTestLog(`${msgPrefix} -- skipping preimages of arbiter node: ${node.host}`);
            continue;
        }

        jsTest.log.info(`${msgPrefix} -- going to check preimages of ${nsUUID} of node: ${node.host}`);
        readers[i] = new ReverseReader(node, node.getDB("config")["system.preimages"], {"_id.nsUUID": nsUUID});
        // Start all reverseReaders at their last document for the collection.
        readers[i].query();
    }

    return readers;
}

function dumpPreImagesCollection(msgPrefix, node, nsUUID, timestamp, limit) {
    const beforeCursor = node
        .getDB("config")
        ["system.preimages"].find({"_id.nsUUID": nsUUID, "_id.ts": {"$lt": timestamp}})
        .sort({$natural: -1})
        .noCursorTimeout()
        .readConcern("local")
        .limit(limit / 2); // We print up to half of the limit in the before part so that
    // the timestamp is centered.
    const beforeEntries = beforeCursor.toArray().reverse();

    let log = `${msgPrefix} -- Dumping a window of ${limit} entries for preimages of collection ${
        nsUUID
    } from host ${node.host} centered around timestamp ${timestamp.toStringIncomparable()}`;

    beforeEntries.forEach((entry) => {
        log += "\n" + tojsononeline(entry);
    });

    const remainingWindow = limit - beforeEntries.length;
    const cursor = node
        .getDB("config")
        ["system.preimages"].find({"_id.nsUUID": nsUUID, "_id.ts": {"$gte": timestamp}})
        .sort({$natural: 1})
        .noCursorTimeout()
        .readConcern("local")
        .limit(remainingWindow);
    cursor.forEach((entry) => {
        log += "\n" + tojsononeline(entry);
    });

    jsTestLog(log);
}

/**
 * Check preimages on all nodes, by reading reading from the last time. Since the preimage may
 * or may not be maintained independently, each node may not contain the same number of entries
 * and stop if the cursor is exhausted on any node being checked.
 *
 * `secondaries` must be the last argument since in checkReplicaSet we explicitly append the live
 * secondaries to the end of the parameter list after ensuring that the current primary is excluded.
 */
function checkPreImageCollection(rst, msgPrefix = "checkPreImageCollection", secondaries) {
    secondaries = secondaries || rst._secondaries;

    const originalPreferences = [];

    jsTest.log.info(`${msgPrefix} -- starting preimage checks.`);
    jsTest.log.info(`${msgPrefix} -- waiting for secondaries to be ready.`);
    rst.awaitSecondaryNodes(rst.timeoutMS, secondaries);
    if (secondaries.length >= 1) {
        let collectionsWithPreimages = {};
        const nodes = rst.nodes;
        for (let i = 0; i < nodes.length; i++) {
            const node = nodes[i];

            if (rst._primary !== node && !secondaries.includes(node)) {
                jsTest.log.info(
                    `${msgPrefix} -- skipping preimages of node as it's not in our list of ` +
                        `secondaries: ${node.host}`,
                );
                continue;
            }

            // Arbiters have no documents in the oplog and thus don't have preimages content.
            if (isNodeArbiter(node)) {
                jsTestLog(`${msgPrefix} -- skipping preimages of arbiter node: ${node.host}`);
                continue;
            }

            const preImageColl = node.getDB("config")["system.preimages"];
            // Reset connection preferences in case the test has modified them. We'll restore
            // them back to what they were originally in the end.
            originalPreferences[i] = {
                secondaryOk: preImageColl.getMongo().getSecondaryOk(),
                readPref: preImageColl.getMongo().getReadPref(),
            };

            preImageColl.getMongo().setSecondaryOk(true);
            preImageColl.getMongo().setReadPref(rst._primary === node ? "primary" : "secondary");

            // Find all collections participating in pre-images.
            const collectionsInPreimages = preImageColl.aggregate([{$group: {_id: "$_id.nsUUID"}}]).toArray();
            for (const collTs of collectionsInPreimages) {
                collectionsWithPreimages[collTs._id] = collTs._id;
            }
        }
        for (const nsUUID of Object.values(collectionsWithPreimages)) {
            const readers = getPreImageReaders(msgPrefix, rst, secondaries, nsUUID);

            while (true) {
                let preImageEntryToCompare = undefined;
                let originNode = undefined;
                for (const reader of readers) {
                    if (reader.hasNext()) {
                        const preImageEntry = reader.next();
                        if (preImageEntryToCompare === undefined) {
                            preImageEntryToCompare = preImageEntry;
                            originNode = reader.mongo;
                        } else {
                            if (!bsonBinaryEqual(preImageEntryToCompare, preImageEntry)) {
                                // TODO SERVER-55756: Investigate if we can remove this since
                                // we'll have the data files present in case this fails with
                                // PeriodicKillSecondaries.
                                jsTest.log.info(`${msgPrefix} -- preimage inconsistency detected.`, {
                                    originNode: {
                                        host: originNode.host,
                                        preImageEntry: preImageEntryToCompare,
                                    },
                                    currentNode: {
                                        host: originNode.host,
                                        preImageEntry: preImageEntryToCompare,
                                    },
                                });
                                jsTest.log.info("Printing previous entries:");
                                dumpPreImagesCollection(
                                    msgPrefix,
                                    originNode,
                                    nsUUID,
                                    preImageEntryToCompare._id.ts,
                                    100,
                                );
                                dumpPreImagesCollection(msgPrefix, reader.mongo, nsUUID, preImageEntry._id.ts, 100);
                                const log =
                                    `${msgPrefix} -- non-matching preimage entries:\n` +
                                    `${originNode.host} -> ${tojsononeline(preImageEntryToCompare)}\n` +
                                    `${reader.mongo.host} -> ${tojsononeline(preImageEntry)}`;
                                assert(false, log);
                            }
                        }
                    }
                }
                if (preImageEntryToCompare === undefined) {
                    break;
                }
            }
        }
    }
    jsTest.log.info(`${msgPrefix} -- preimages check complete.`);

    // Restore original read preferences used by the connection.
    for (const idx in originalPreferences) {
        const node = rst.nodes[idx];
        const conn = node.getDB("config").getMongo();
        conn.setSecondaryOk(originalPreferences[idx].secondaryOk);
        conn.setReadPref(originalPreferences[idx].readPref);
    }
}

function dumpChangeCollection(node, tenantDatabaseName, timestamp, limit, msgPrefix) {
    const beforeCursor = node
        .getDB(tenantDatabaseName)
        ["system.change_collection"].find({"_id": {"$lt": timestamp}})
        .sort({$natural: -1})
        .noCursorTimeout()
        .readConcern("local")
        .limit(limit / 2); // We print up to half of the limit in the before part so that
    // the timestamp is centered.
    const beforeEntries = beforeCursor.toArray().reverse();

    let log = `${msgPrefix} -- Dumping a window of ${limit} entries for ${
        tenantDatabaseName
    }.system.change_collection from host ${node.host} centered around ${timestamp.toStringIncomparable()}`;

    beforeEntries.forEach((entry) => {
        log += "\n" + tojsononeline(entry);
    });

    const remainingWindow = limit - beforeEntries.length;
    const cursor = node
        .getDB(tenantDatabaseName)
        ["system.change_collection"].find({"_id": {"$gte": timestamp}})
        .sort({$natural: 1})
        .noCursorTimeout()
        .readConcern("local")
        .limit(remainingWindow);
    cursor.forEach((entry) => {
        log += "\n" + tojsononeline(entry);
    });

    jsTestLog(log);
}

function checkTenantChangeCollection(rst, secondaries, db, msgPrefix = "checkTenantChangeCollection") {
    const tenantDatabaseName = db.name;
    jsTest.log.info(`${msgPrefix} -- starting check on ${db.tenantId} ${tenantDatabaseName}.system.change_collection`);

    // Prepare reverse read from the primary and specified secondaries.
    const nodes = [rst.getPrimary(), ...secondaries];
    let reverseReaders = nodes.map((node) => {
        let reader = new ReverseReader(node, node.getDB(tenantDatabaseName)["system.change_collection"]);
        // Start all reverseReaders at their last document for the collection.
        reader.query();
        return reader;
    });

    let inspectedEntryCount = 0;
    while (true) {
        const entryAndNodeSet = reverseReaders.map((reader) => {
            if (reader.hasNext()) {
                return {entry: reader.next(), node: reader.mongo};
            }
            return undefined;
        });
        let baselineEntryAndNode = undefined;

        entryAndNodeSet.forEach((entryAndNode) => {
            if (entryAndNode === undefined) {
                return;
            }

            if (baselineEntryAndNode === undefined) {
                inspectedEntryCount++;
                baselineEntryAndNode = entryAndNode;
                return;
            }
            if (!bsonBinaryEqual(baselineEntryAndNode.entry, entryAndNode.entry)) {
                jsTest.log.info(
                    `${msgPrefix} -- inconsistency detected in ${tenantDatabaseName}.system.change_collection`,
                    {
                        baselineNode: {
                            host: baselineEntryAndNode.node.host,
                            entry: baselineEntryAndNode.entry,
                        },
                        currentNode: {host: entryAndNode.node.host, entry: entryAndNode.entry},
                    },
                );

                dumpChangeCollection(
                    baselineEntryAndNode.node,
                    tenantDatabaseName,
                    baselineEntryAndNode.entry._id,
                    100,
                    msgPrefix,
                );
                dumpChangeCollection(entryAndNode.node, tenantDatabaseName, entryAndNode.entry._id, 100, msgPrefix);
                assert(false, `Found inconsistency in '${tenantDatabaseName}.system.change_collection'`);
            }
        });

        if (baselineEntryAndNode === undefined) {
            break;
        }
    }
    jsTest.log.info(
        `${msgPrefix} -- finished check on ${tenantDatabaseName}.system.change_collection, inspected ${
            inspectedEntryCount
        } unique entries`,
    );
}

/**
 * Check change_collection for all tenants on all nodes, by doing a reverse scan. This check
 * accounts for the fact that each node might independently truncate the change collection, and
 * not contain the same number of entries.
 *
 * `secondaries` must be the last argument since in checkReplicaSet we explicitly append the live
 * secondaries to the end of the parameter list after ensuring that the current primary is excluded.
 */
function checkChangeCollection(rst, msgPrefix = "checkChangeCollection", secondaries) {
    secondaries = secondaries || rst._secondaries;
    secondaries = secondaries.filter((node) => !isNodeArbiter(node));

    if (secondaries.length == 0) {
        jsTest.log.info(`${msgPrefix} -- no data bearing secondaries specified, nothing to do.`);
        return;
    }

    jsTest.log.info(`${msgPrefix} -- starting change_collection checks.`);
    jsTest.log.info(`${msgPrefix} -- waiting for secondaries to be ready.`);
    rst.awaitSecondaryNodes(rst.timeoutMS, secondaries);

    // Get all change_collections for all tenants.
    let dbs = rst.getPrimary().getDBs();
    dbs = dbs.databases.filter((db) => db.name.endsWith("_config") || db.name == "config");
    dbs.forEach((db) => {
        if (db.tenantId) {
            try {
                const token = _createTenantToken({tenant: db.tenantId});
                rst.nodes.forEach((node) => node._setSecurityToken(token));
                checkTenantChangeCollection(rst, secondaries, db);
            } finally {
                rst.nodes.forEach((node) => node._setSecurityToken(undefined));
            }
        } else {
            checkTenantChangeCollection(rst, secondaries, db);
        }
    });
    jsTest.log.info(`${msgPrefix} -- change_collection check complete.`);
}

/**
 * Recursively merge the target and source object.
 */
function _deepObjectMerge(target, source) {
    if (!(target instanceof Object)) {
        return source === undefined || source === null ? target : source;
    }

    if (!(source instanceof Object)) {
        return target;
    }

    let res = Object.assign({}, target);
    Object.keys(source).forEach((k) => {
        res[k] = _deepObjectMerge(target[k], source[k]);
    });

    return res;
}

/**
 * Resolves a parameter into a direct connection to a replica set node.
 *
 * @param {ReplSetTest} rst The ReplicaSetTest we are resolving the parameter for.
 * @param {number|Mongo} nodeIdOrConnection The parameter we want to resolve into a connection.
 * @returns {Mongo}
 */
function resolveToConnection(rst, nodeIdOrConnection) {
    if (nodeIdOrConnection.getDB) {
        return nodeIdOrConnection;
    }

    assert(rst.nodes.hasOwnProperty(nodeIdOrConnection), `${nodeIdOrConnection} not found in own nodes`);
    return rst.nodes[nodeIdOrConnection];
}

/**
 * Resolves a parameter into a replica set node id.
 *
 * @param {ReplicaSetTest} rst The ReplicaSetTest we are resolving the parameter for.
 * @param {number|Mongo} nodeIdOrConnection The parameter we want to resolve into a node id.
 * @returns {number}
 */
function resolveToNodeId(rst, nodeIdOrConnection) {
    if (nodeIdOrConnection.getDB) {
        return rst.getNodeId(nodeIdOrConnection);
    }

    assert(Number.isInteger(nodeIdOrConnection), `node must be an integer, not ${nodeIdOrConnection}`);
    return nodeIdOrConnection;
}