mirror of https://github.com/mongodb/mongo
98 lines
3.0 KiB
JavaScript
98 lines
3.0 KiB
JavaScript
/**
|
|
* Tests that if a node is in catchup for too long (before exiting drain mode),
|
|
* and another node takes over and becomes primary, that the former node will
|
|
* safely exit drain mode and step down.
|
|
*
|
|
* @tags: [
|
|
* requires_replication,
|
|
* ]
|
|
*/
|
|
|
|
import {configureFailPoint} from "jstests/libs/fail_point_util.js";
|
|
import {ReplSetTest} from "jstests/libs/replsettest.js";
|
|
|
|
const dbName = "test";
|
|
const collName = "coll";
|
|
|
|
const rst = new ReplSetTest({
|
|
nodes: [
|
|
{},
|
|
{},
|
|
// Prevent node 2 from trying to run for election.
|
|
{rsConfig: {priority: 0}},
|
|
],
|
|
useBridge: true,
|
|
settings: {chainingAllowed: true, catchupTimeoutMillis: -1},
|
|
});
|
|
|
|
const nodes = rst.startSet();
|
|
assert.eq(nodes.length, 3);
|
|
rst.initiate();
|
|
const dbs = nodes.map((node) => node.getDB(dbName));
|
|
const colls = dbs.map((db) => db.getCollection(collName));
|
|
|
|
function insertDocument(db) {
|
|
return db.runCommand({
|
|
insert: collName,
|
|
documents: [{foo: "bar"}],
|
|
writeConcern: {w: 1},
|
|
});
|
|
}
|
|
|
|
// Make node 1 the primary.
|
|
rst.stepUp(nodes[1]);
|
|
|
|
// Ensure that all nodes are in the same term before enabling failpoints.
|
|
rst.awaitReplication();
|
|
|
|
// Set up a stopReplProducer failpoint on nodes 0 and 2.
|
|
// Node 0 needs to be behind node 1 in order for it to enter drain mode.
|
|
// Node 2 cannot be ahead of node 0 in order for node 0 to win its election.
|
|
const node0SrpFailpoint = configureFailPoint(nodes[0], "stopReplProducer");
|
|
const node2SrpFailpoint = configureFailPoint(nodes[2], "stopReplProducer");
|
|
|
|
// Insert a document on node 1 with w: 1.
|
|
const insertResponse = insertDocument(dbs[1]);
|
|
const insertTimestamp = insertResponse.operationTime;
|
|
|
|
// Set up a hangBeforeRSTLOnDrainComplete failpoint on node 0 to make it hang
|
|
// during drain mode.
|
|
const node0DrainModeFailpoint = configureFailPoint(nodes[0], "hangBeforeRSTLOnDrainComplete", {}, "alwaysOn");
|
|
|
|
// Wait for node 0 to become aware of the insert via heartbeats.
|
|
assert.soon(() => {
|
|
const replSetGetStatus = assert.commandWorked(nodes[0].adminCommand({replSetGetStatus: 1}));
|
|
return bsonWoCompare(replSetGetStatus.$clusterTime.clusterTime, insertTimestamp) >= 0;
|
|
});
|
|
|
|
// Step up node 0 in a parallel shell.
|
|
// The shell will join after catchup completes, but before drain mode completes.
|
|
const shell = startParallelShell(
|
|
() => assert.soonNoExcept(() => db.adminCommand({replSetStepUp: 1}).ok),
|
|
nodes[0].port,
|
|
);
|
|
shell();
|
|
|
|
// Wait for node 0 to enter drain mode.
|
|
node0SrpFailpoint.off();
|
|
node0DrainModeFailpoint.wait();
|
|
|
|
// Assert that node 0 was able to successfully replicate the insert.
|
|
assert.eq("bar", colls[0].find({}).readConcern("local").toArray()[0].foo);
|
|
|
|
// Step up node 1.
|
|
rst.stepUp(nodes[1], {awaitReplicationBeforeStepUp: false});
|
|
rst.awaitNodesAgreeOnPrimary(rst.timeoutMS, nodes, nodes[1]);
|
|
|
|
// Disable the failpoints on node 0.
|
|
node0DrainModeFailpoint.off();
|
|
|
|
// Ensure that node 0 has stepped down.
|
|
const res = nodes[0].adminCommand({hello: 1});
|
|
assert.eq(res.isWritablePrimary, false);
|
|
assert.eq(res.secondary, true);
|
|
|
|
node2SrpFailpoint.off();
|
|
|
|
rst.stopSet();
|