mongo/jstests/sharding/secondary_reads_waitForShardingInit.js

/**
 * Tests that a secondary that hasn't finished sharding initialization can successfully handle
 * a read request that uses afterClusterTime >= the optime of the insert of the shard identity
 * document on the primary.

 * @tags: [
 *     requires_fcv_52,
 *     # This test is incompatible with 'config shard' as it creates a cluster with 0 shards in
 *     # order to be able to add shard with data on it (which is only allowed on the first shard).
 *     config_shard_incompatible,
 * ]
 */

import {configureFailPoint} from "jstests/libs/fail_point_util.js";
import {FeatureFlagUtil} from "jstests/libs/feature_flag_util.js";
import {ReplSetTest} from "jstests/libs/replsettest.js";
import {ShardingTest} from "jstests/libs/shardingtest.js";
import {forceSyncSource} from "jstests/replsets/libs/sync_source.js";

let st = new ShardingTest({shards: 0});

// Set up replica set that we'll add as shard
let replTest = new ReplSetTest({nodes: 3, name: jsTest.name() + "-newReplSet"});
replTest.startSet({shardsvr: ""});
let nodeList = replTest.nodeList();

replTest.initiate({
    _id: replTest.name,
    members: [
        {_id: 0, host: nodeList[0], priority: 1},
        {_id: 1, host: nodeList[1], priority: 0, tags: {"tag": "hanging"}},
        {_id: 2, host: nodeList[2], priority: 0},
    ],
});

let primary = replTest.getPrimary();
let hangingSecondary = replTest.getSecondaries()[0];
let anotherSecondary = replTest.getSecondaries()[1];

// Set failpoint on one secondary to hang during sharding initialization
jsTest.log("Going to turn on the hangDuringShardingInitialization fail point.");
const fpHangDuringShardInit = configureFailPoint(hangingSecondary, "hangDuringShardingInitialization");

/**
 * Force the other secondary node to sync from the primary. If it syncs from the hanging secondary
 * node that has not inserted the sharding identity document, the sharding initialization will not
 * be triggered and the addShard command will fail
 */
jsTest.log("Going to set sync source of secondary node to be primary.");
const fpForceSyncSource = forceSyncSource(replTest, anotherSecondary, primary);

jsTest.log("Going to add replica set as shard: " + tojson(replTest.getReplSetConfig()));
const shardName = "newShard";
assert.commandWorked(st.s.getDB("admin").runCommand({addShard: replTest.getURL(), name: shardName}));
fpHangDuringShardInit.wait();

jsTest.log("Check and wait for the sharding state to be initialized on primary.");
assert.soon(function () {
    const shardingStatePrimary = replTest.getPrimary().getDB("admin").runCommand({shardingState: 1});
    return shardingStatePrimary.enabled == true;
});

const dbName = "testDB";
const sessionDb = st.s.startSession().getDatabase(dbName);

jsTest.log("Going to write a document to testDB.foo.");
// Make sure that the test db data is stored into the new shard.
assert.commandWorked(st.s.adminCommand({enableSharding: dbName, primaryShard: shardName}));
assert.commandWorked(sessionDb.foo.insert({x: 1}));

// TODO (SERVER-97816) remove check once 9.0 becomes last LTS.
const isMultiversion = Boolean(jsTest.options().useRandomBinVersionsWithinReplicaSet);
if (!isMultiversion) {
    jsTest.log("Going to send a read request which will be versioned by the mongoS to force the secondary to wait");
    const error = st.s.getDB(dbName).runCommand({
        find: "foo",
        maxTimeMS: 10000,
        $readPreference: {mode: "secondary", tags: [{"tag": "hanging"}]},
    });
    assert.commandFailedWithCode(error, ErrorCodes.MaxTimeMSExpired);
}

/**
 * Send a read request to the hanging secondary node. We expect it to fail as the sharding state is
 * not initialized. The read will block waiting for the afterClusterTime that is younger (greater)
 * than the opLog timestamp on hanging secondary node. A maxTimeMs is set to force the read time out
 * so that it will not block for too long.
 *
 * In this test case
 * -T1: insert of the shard identity doc (addShard)
 * -T2: write operation (insert)

 * The afterClusterTime we give is T2.  On hanging secondary node, the oplog is still at ts < T1.
 */
jsTest.log(
    "Going to send a read request with maxTimeMS 100000 to secondary that is hanging in setting up sharding initialization.",
);
const operationTime = sessionDb.getSession().getOperationTime();
const error = sessionDb.runCommand({
    find: "foo",
    maxTimeMS: 10000,
    $readPreference: {mode: "secondary", tags: [{"tag": "hanging"}]},
    readConcern: {level: "local", "afterClusterTime": operationTime},
});
assert.commandFailedWithCode(error, ErrorCodes.MaxTimeMSExpired);

jsTest.log("Going to turn off the hangDuringShardingInitialization.");
fpHangDuringShardInit.off();

jsTest.log("Check and wait for the sharding state to be initialized on hanging secondary.");
assert.soon(
    function () {
        return hangingSecondary.getDB("admin").runCommand({shardingState: 1}).enabled == true;
    },
    "Mongos did not update its sharding state after 10 seconds",
    10 * 1000,
);

/**
 * Send the read request again. We expect it to succeed now as the sharding state is initialized and
 * the read won't block waiting for read concern.
 */
jsTest.log("Going to send the read request again.");
assert.commandWorked(
    sessionDb.runCommand({
        find: "foo",
        $readPreference: {mode: "secondary", tags: [{"tag": "hanging"}]},
        readConcern: {level: "local", "afterClusterTime": operationTime},
    }),
);

fpForceSyncSource.off();
sessionDb.getSession().endSession();

replTest.stopSet();
st.stop();