mongo/jstests/replsets/docs_initialsync_workflow.js

/**
 * This test simulates workflows for adding a new node and resyncing a node recommended by docs.
 */
import {ReplSetTest} from "jstests/libs/replsettest.js";
import {disconnectSecondaries, reconnectSecondaries, waitForState} from "jstests/replsets/rslib.js";

const testName = TestData.testName;
const rst = new ReplSetTest({
    name: testName,
    nodes: [{}, {rsConfig: {priority: 0}}, {rsConfig: {priority: 0}}],
    useBridge: true,
    // We shorten the election timeout period so the tests with an unhealthy set run and recover
    // faster.
    settings: {electionTimeoutMillis: 2000, heartbeatIntervalMillis: 400},
});
rst.startSet();
rst.initiate();
// Add some data.
const primary = rst.getPrimary();
const testDb = primary.getDB("test");
assert.commandWorked(testDb[testName].insert([{a: 1}, {b: 2}, {c: 3}]));
rst.awaitReplication();

function testAddWithInitialSync(secondariesDown) {
    let config = rst.getReplSetConfigFromNode();
    disconnectSecondaries(rst, secondariesDown);
    const majorityDown = secondariesDown > 1;
    if (majorityDown) {
        // Wait for the set to become unhealthy.
        rst.awaitSecondaryNodes(null, [primary]);
    }
    // Add a new, voting node.
    const newNode = rst.add({rsConfig: {priority: 0}});
    // The second disconnect ensures we can't reach the new node from the 'down' nodes.
    disconnectSecondaries(rst, secondariesDown);
    const newConfig = rst.getReplSetConfig();
    config.members = newConfig.members;
    config.version += 1;
    jsTestLog("Reconfiguring set to add node.");
    assert.commandWorked(
        primary.adminCommand({replSetReconfig: config, maxTimeMS: ReplSetTest.kDefaultTimeoutMS, force: majorityDown}),
    );

    jsTestLog("Waiting for node to sync.");
    rst.awaitSecondaryNodes(null, [newNode]);

    // Make sure the set is still consistent after adding the node.
    reconnectSecondaries(rst);
    // If we were in a majority-down scenario, wait for the primary to be re-elected.
    assert.soon(() => primary == rst.getPrimary());
    // Wait for the automatic reconfig to complete.
    rst.waitForAllNewlyAddedRemovals();

    rst.checkOplogs();
    rst.checkReplicatedDataHashes();

    // Remove our extra node.
    rst.stop(newNode);
    rst.remove(newNode);
    rst.reInitiate();
}

function testReplaceWithInitialSync(secondariesDown) {
    // Wait for existing reconfigs to complete.
    rst.waitForAllNewlyAddedRemovals();

    const nodeToBeReplaced = rst.getSecondaries()[2];
    const majorityDown = secondariesDown > 1;
    let config = rst.getReplSetConfigFromNode(primary.nodeId);
    disconnectSecondaries(rst, secondariesDown);
    if (majorityDown) {
        // Wait for the set to become unhealthy.
        rst.awaitSecondaryNodes(null, [primary]);
    }

    let nodeId = rst.getNodeId(nodeToBeReplaced);
    const highestMemberId = config.members[nodeId]._id;

    jsTestLog("Reconfiguring to remove the node.");
    config.version += 1;
    config.members.splice(nodeId, 1);
    assert.commandWorked(
        primary.adminCommand({replSetReconfig: config, maxTimeMS: ReplSetTest.kDefaultTimeoutMS, force: majorityDown}),
    );

    jsTestLog("Stopping node for replacement");
    rst.stop(nodeToBeReplaced, undefined, {skipValidation: true}, {forRestart: true});
    rst.remove(nodeToBeReplaced);
    if (!majorityDown) {
        // Add some data.  This can't work in a majority-down situation, so we don't do it then.
        assert.commandWorked(
            testDb[testName].insert({replaceWithInitialSync: secondariesDown}, {writeConcern: {w: 1}}),
        );
    }

    jsTestLog("Starting a new replacement node with empty data directory.");
    const replacementNode = rst.add({rsConfig: {priority: 0}});
    // The second disconnect ensures we can't reach the replacement node from the 'down' nodes.
    disconnectSecondaries(rst, secondariesDown);

    nodeId = rst.getNodeId(replacementNode);
    config = rst.getReplSetConfigFromNode(primary.nodeId);
    const newConfig = rst.getReplSetConfig();
    config.members = newConfig.members;
    // Don't recycle the member id.
    config.members[nodeId]._id = highestMemberId + 1;
    config.version += 1;
    jsTestLog("Reconfiguring set to add the replacement node.");
    assert.commandWorked(
        primary.adminCommand({replSetReconfig: config, maxTimeMS: ReplSetTest.kDefaultTimeoutMS, force: majorityDown}),
    );

    jsTestLog("Waiting for the replacement node to sync.");
    rst.awaitSecondaryNodes(null, [replacementNode]);

    if (!majorityDown) {
        // Make sure we can replicate to it, if the set is otherwise healthy.
        rst.awaitReplication(undefined, undefined, [replacementNode]);
    }
    // Make sure the set is still consistent after resyncing the node.
    reconnectSecondaries(rst);
    // If we were in a majority-down scenario, wait for the primary to be re-elected.
    assert.soon(() => primary == rst.getPrimary());
    rst.checkOplogs();
    rst.checkReplicatedDataHashes();
}

jsTestLog("Test adding a node with initial sync in a healthy system.");
testAddWithInitialSync(0);

jsTestLog("Test adding a node with initial sync with one secondary unreachable.");
testAddWithInitialSync(1);

jsTestLog("Test adding a node with initial sync with two secondaries unreachable.");
testAddWithInitialSync(2);

jsTestLog("Adding node for replace-node scenarios");
const newNode = rst.add({rsConfig: {priority: 0}});
rst.reInitiate();
// Wait for the node to to become secondary.
waitForState(newNode, ReplSetTest.State.SECONDARY);

jsTestLog("Test replacing a node with initial sync in a healthy system.");
testReplaceWithInitialSync(0);

jsTestLog("Test replacing a node with initial sync with one secondary unreachable.");
testReplaceWithInitialSync(1);

jsTestLog("Test replacing a node with initial sync with two secondaries unreachable.");
testReplaceWithInitialSync(2);

rst.stopSet();