mongo/jstests/replsets/linearizable_read_concern.js

/*
 * This test creates a 3 node replica set. The test first sends
 * a regular linearizable read command which should succeed. Then the test
 * examines linearizable read parsing abilities by sending a linearizable
 * read command to a secondary and then to the primary with an 'afterOpTime'
 * field, both of which should fail. The test then starts to test the actual
 * functionality of linearizable reads by creating a network partition between the primary
 * and the other two nodes and then sending in a linearizable read command.
 * Finally we test whether the linearizable read command will block forever
 * by issuing a linearizable read command in another thread on the still
 * partitioned primary and then making the primary step down in the main
 * thread after finding the issued noop. The secondary thread should throw
 * an exception and exit.
 */
import {ReplSetTest} from "jstests/libs/replsettest.js";
import {getLatestOp} from "jstests/replsets/rslib.js";

let send_linearizable_read = function () {
    // The primary will step down and throw an exception, which is expected.
    let coll = db.getSiblingDB("test").foo;
    jsTestLog("Sending in linearizable read in secondary thread");
    // 'hello' ensures that the following command fails (and returns a response rather than
    // an exception) before its connection is cut because of the primary step down. Refer to
    // SERVER-24574.
    assert.commandWorked(coll.runCommand({hello: 1, hangUpOnStepDown: false}));
    assert.commandFailedWithCode(
        coll.runCommand({"find": "foo", readConcern: {level: "linearizable"}, maxTimeMS: 60000}),
        ErrorCodes.InterruptedDueToReplStateChange,
    );
};

let num_nodes = 3;
let name = "linearizable_read_concern";
let replTest = new ReplSetTest({name: name, nodes: num_nodes, useBridge: true});
let config = replTest.getReplSetConfig();

// Increased election timeout to avoid having the primary step down while we are
// testing linearizable functionality on an isolated primary.
config.settings = {
    electionTimeoutMillis: 60000,
};

replTest.startSet();
replTest.initiate(config);

let primary = replTest.getPrimary();
let secondaries = replTest.getSecondaries();

// The default WC is majority and this test can't satisfy majority writes.
assert.commandWorked(
    primary.adminCommand({setDefaultRWConcern: 1, defaultWriteConcern: {w: 1}, writeConcern: {w: "majority"}}),
);

// Without a sync source the heartbeat interval will be half of the election timeout, 30
// seconds. It thus will take almost 30 seconds for the secondaries to set the primary as
// their sync source and begin replicating.
replTest.awaitReplication();
// Do a write to have something to read.
assert.commandWorked(
    primary
        .getDB("test")
        .foo.insert({"number": 7}, {"writeConcern": {"w": "majority", "wtimeout": ReplSetTest.kDefaultTimeoutMS}}),
);

jsTestLog("Testing linearizable readConcern parsing");
// This command is sent to the primary, and the primary is fully connected so it should work.
let goodRead = assert.commandWorked(
    primary.getDB("test").runCommand({"find": "foo", readConcern: {level: "linearizable"}, "maxTimeMS": 60000}),
);
assert.eq(goodRead.cursor.firstBatch[0].number, 7);

// This fails because you cannot have a linearizable read command sent to a secondary.
let badCmd = assert.commandFailed(
    secondaries[0].getDB("test").runCommand({"find": "foo", readConcern: {level: "linearizable"}, "maxTimeMS": 60000}),
);

assert.eq(badCmd.errmsg, "cannot satisfy linearizable read concern on non-primary node");
assert.eq(badCmd.code, ErrorCodes.NotWritablePrimary);

// This fails because you cannot specify 'afterOpTime' for linearizable read.
let opTimeCmd = assert.commandFailed(
    primary.getDB("test").runCommand({
        "find": "foo",
        readConcern: {level: "linearizable", "afterOpTime": {ts: Timestamp(1, 2), t: 1}},
        "maxTimeMS": 60000,
    }),
);
assert.eq(opTimeCmd.errmsg, "afterOpTime not compatible with linearizable read concern");
assert.eq(opTimeCmd.code, ErrorCodes.FailedToParse);

// A $out aggregation is not allowed with readConcern level "linearizable".
assert.throwsWithCode(
    () => primary.getDB("test").foo.aggregate([{$out: "out"}], {readConcern: {level: "linearizable"}}),
    ErrorCodes.InvalidOptions,
);

// A $merge aggregation is not allowed with readConcern level "linearizable".
assert.throwsWithCode(
    () =>
        primary
            .getDB("test")
            .foo.aggregate([{$merge: {into: "out", whenMatched: "replace", whenNotMatched: "insert"}}], {
                readConcern: {level: "linearizable"},
            }),
    ErrorCodes.InvalidOptions,
);

primary = replTest.getPrimary();

jsTestLog("Starting linearizablility testing");

const cursorId = assert.commandWorked(
    primary.getDB("test").runCommand({"find": "foo", readConcern: {level: "linearizable"}, batchSize: 0}),
).cursor.id;
jsTestLog("Setting up partitions such that the primary is isolated: [Secondary-Secondary] [Primary]");
secondaries[0].disconnect(primary);
secondaries[1].disconnect(primary);

jsTestLog("Testing to make sure that linearizable getMores will time out when the primary is isolated.");
assert.commandWorked(primary.getDB("test").foo.insert({_id: 0, x: 0}));
assert.commandFailedWithCode(
    primary.getDB("test").runCommand({"getMore": cursorId, collection: "foo", batchSize: 1}),
    ErrorCodes.LinearizableReadConcernError,
);

jsTestLog("Test that a linearizable read will timeout when the primary is isolated.");
let findResult = primary
    .getDB("test")
    .runCommand({"find": "foo", "readConcern": {level: "linearizable"}, "maxTimeMS": 3000});
assert.commandFailedWithCode(findResult, ErrorCodes.MaxTimeMSExpired);

jsTestLog("Testing to make sure linearizable read command does not block forever.");

// Get last noop Optime before sending the linearizable read command
// to ensure that we are waiting for the most recent noop write.
let lastOpTimestamp = getLatestOp(primary).ts;

let parallelShell = startParallelShell(send_linearizable_read, primary.port);
// Sending a linearizable read implicitly replicates a noop to the secondaries. We need to find
// the most recently issued noop to ensure that we call stepdown during the recently
// issued linearizable read and not before the read (in the separate thread) has been called.
jsTestLog("Checking end of oplog for noop");
assert.soon(function () {
    let isEarlierTimestamp = function (ts1, ts2) {
        if (ts1.getTime() == ts2.getTime()) {
            return ts1.getInc() < ts2.getInc();
        }
        return ts1.getTime() < ts2.getTime();
    };
    let latestOp = getLatestOp(primary);
    if (latestOp.op == "n" && isEarlierTimestamp(lastOpTimestamp, latestOp.ts)) {
        return true;
    }

    return false;
});
assert.eq(primary, replTest.getPrimary(), "Primary unexpectedly changed mid test.");
jsTestLog("Making Primary step down");
assert.commandWorked(primary.adminCommand({"replSetStepDown": 100, secondaryCatchUpPeriodSecs: 0, "force": true}));
parallelShell();
replTest.stopSet();