mongo/jstests/replsets/linearizable_read_concern.js

/*
 * This test creates a 3 node replica set. The test first sends
 * a regular linearizable read command which should succeed. Then the test
 * examines linearizable read parsing abilities by sending a linearizable
 * read command to a secondary and then to the primary with an 'afterOpTime'
 * field, both of which should fail. The test then starts to test the actual
 * functionality of linearizable reads by creating a network partition between the primary
 * and the other two nodes and then sending in a linearizable read command.
 * Finally we test whether the linearizable read command will block forever
 * by issuing a linearizable read command in another thread on the still
 * partitioned primary and then making the primary step down in the main
 * thread after finding the issued noop. The secondary thread should throw
 * an exception and exit.
 */
load('jstests/replsets/rslib.js');
load('jstests/libs/parallelTester.js');
load('jstests/libs/write_concern_util.js');
(function() {
'use strict';
var send_linearizable_read = function() {
    // The primary will step down and throw an exception, which is expected.
    var coll = db.getSiblingDB("test").foo;
    jsTestLog('Sending in linearizable read in secondary thread');
    // 'hello' ensures that the following command fails (and returns a response rather than
    // an exception) before its connection is cut because of the primary step down. Refer to
    // SERVER-24574.
    assert.commandWorked(coll.runCommand({hello: 1, hangUpOnStepDown: false}));
    assert.commandFailedWithCode(
        coll.runCommand({'find': 'foo', readConcern: {level: "linearizable"}, maxTimeMS: 60000}),
        ErrorCodes.InterruptedDueToReplStateChange);
};

var num_nodes = 3;
var name = 'linearizable_read_concern';
var replTest = new ReplSetTest({name: name, nodes: num_nodes, useBridge: true});
var config = replTest.getReplSetConfig();

// Increased election timeout to avoid having the primary step down while we are
// testing linearizable functionality on an isolated primary.
config.settings = {
    electionTimeoutMillis: 60000
};

replTest.startSet();
replTest.initiate(config);

var primary = replTest.getPrimary();
var secondaries = replTest.getSecondaries();

// The default WC is majority and this test can't satisfy majority writes.
assert.commandWorked(primary.adminCommand(
    {setDefaultRWConcern: 1, defaultWriteConcern: {w: 1}, writeConcern: {w: "majority"}}));

// Without a sync source the heartbeat interval will be half of the election timeout, 30
// seconds. It thus will take almost 30 seconds for the secondaries to set the primary as
// their sync source and begin replicating.
replTest.awaitReplication();
// Do a write to have something to read.
assert.commandWorked(primary.getDB("test").foo.insert(
    {"number": 7}, {"writeConcern": {"w": "majority", "wtimeout": ReplSetTest.kDefaultTimeoutMS}}));

jsTestLog("Testing linearizable readConcern parsing");
// This command is sent to the primary, and the primary is fully connected so it should work.
var goodRead = assert.commandWorked(primary.getDB("test").runCommand(
    {'find': 'foo', readConcern: {level: "linearizable"}, "maxTimeMS": 60000}));
assert.eq(goodRead.cursor.firstBatch[0].number, 7);

// This fails because you cannot have a linearizable read command sent to a secondary.
var badCmd = assert.commandFailed(secondaries[0].getDB("test").runCommand(
    {"find": "foo", readConcern: {level: "linearizable"}, "maxTimeMS": 60000}));

assert.eq(badCmd.errmsg, "cannot satisfy linearizable read concern on non-primary node");
assert.eq(badCmd.code, ErrorCodes.NotWritablePrimary);

// This fails because you cannot specify 'afterOpTime' for linearizable read.
var opTimeCmd = assert.commandFailed(primary.getDB("test").runCommand({
    "find": "foo",
    readConcern: {level: "linearizable", "afterOpTime": {ts: Timestamp(1, 2), t: 1}},
    "maxTimeMS": 60000
}));
assert.eq(opTimeCmd.errmsg, "afterOpTime not compatible with linearizable read concern");
assert.eq(opTimeCmd.code, ErrorCodes.FailedToParse);

// A $out aggregation is not allowed with readConcern level "linearizable".
assert.throwsWithCode(() => primary.getDB("test").foo.aggregate(
                          [{$out: "out"}], {readConcern: {level: "linearizable"}}),
                      ErrorCodes.InvalidOptions);

// A $merge aggregation is not allowed with readConcern level "linearizable".
assert.throwsWithCode(
    () => primary.getDB("test").foo.aggregate(
        [{$merge: {into: "out", whenMatched: "replace", whenNotMatched: "insert"}}],
        {readConcern: {level: "linearizable"}}),
    ErrorCodes.InvalidOptions);

primary = replTest.getPrimary();

jsTestLog("Starting linearizablility testing");

const cursorId = assert
                     .commandWorked(primary.getDB("test").runCommand(
                         {'find': 'foo', readConcern: {level: "linearizable"}, batchSize: 0}))
                     .cursor.id;
jsTestLog(
    "Setting up partitions such that the primary is isolated: [Secondary-Secondary] [Primary]");
secondaries[0].disconnect(primary);
secondaries[1].disconnect(primary);

jsTestLog(
    "Testing to make sure that linearizable getMores will time out when the primary is isolated.");
assert.commandWorked(primary.getDB("test").foo.insert({_id: 0, x: 0}));
assert.commandFailedWithCode(
    primary.getDB("test").runCommand({"getMore": cursorId, collection: "foo", batchSize: 1}),
    ErrorCodes.LinearizableReadConcernError);

jsTestLog("Test that a linearizable read will timeout when the primary is isolated.");
let findResult = primary.getDB("test").runCommand(
    {"find": "foo", "readConcern": {level: "linearizable"}, "maxTimeMS": 3000});
assert.commandFailedWithCode(findResult, ErrorCodes.MaxTimeMSExpired);

jsTestLog("Testing to make sure linearizable read command does not block forever.");

// Get last noop Optime before sending the linearizable read command
// to ensure that we are waiting for the most recent noop write.
var lastOpTimestamp = getLatestOp(primary).ts;

var parallelShell = startParallelShell(send_linearizable_read, primary.port);
// Sending a linearizable read implicitly replicates a noop to the secondaries. We need to find
// the most recently issued noop to ensure that we call stepdown during the recently
// issued linearizable read and not before the read (in the separate thread) has been called.
jsTestLog("Checking end of oplog for noop");
assert.soon(function() {
    var isEarlierTimestamp = function(ts1, ts2) {
        if (ts1.getTime() == ts2.getTime()) {
            return ts1.getInc() < ts2.getInc();
        }
        return ts1.getTime() < ts2.getTime();
    };
    var latestOp = getLatestOp(primary);
    if (latestOp.op == "n" && isEarlierTimestamp(lastOpTimestamp, latestOp.ts)) {
        return true;
    }

    return false;
});
assert.eq(primary, replTest.getPrimary(), "Primary unexpectedly changed mid test.");
jsTestLog("Making Primary step down");
assert.commandWorked(
    primary.adminCommand({"replSetStepDown": 100, secondaryCatchUpPeriodSecs: 0, "force": true}));
parallelShell();
replTest.stopSet();
}());