/** * Tests that when the 'movePrimaryFailIfNeedToCloneMovableCollections' failpoint is enabled, a * movePrimary command would only fail if: * - The command is run with the same 'comment' as specified in the failpoint. * - The donor shard still has user data for that database (i.e. untracked unsharded * collections that are movable by moveCollection). * * @tags: [requires_fcv_81] */ import {EncryptedClient} from "jstests/fle2/libs/encrypted_client_util.js"; import {configureFailPoint} from "jstests/libs/fail_point_util.js"; import {ShardingTest} from "jstests/libs/shardingtest.js"; import {extractUUIDFromObject} from "jstests/libs/uuid_util.js"; function getCollectionUuid(db, collName) { const listCollectionRes = assert.commandWorked(db.runCommand({listCollections: 1, filter: {name: collName}})); return listCollectionRes.cursor.firstBatch[0].info.uuid; } function createCollection(st, dbName, collName, track) { if (track) { st.s.getDB(dbName).runCommand({createUnsplittableCollection: collName}); } else { st.s.getDB(dbName).runCommand({create: collName}); } } function testUserCollections({trackUnshardedCollections}) { jsTest.log("Running tests for user collections " + tojson({trackUnshardedCollections})); const st = new ShardingTest({ shards: 3, configShard: true, }); const shard0Primary = st.rs0.getPrimary(); const shard1Primary = st.rs1.getPrimary(); const dbName0 = "testDb0"; const dbName1 = "testDb1"; const dbName2 = "testDb2"; const collName0 = "testColl0"; const collName1 = "testColl1"; jsTest.log("Testing 'movePrimaryFailIfNeedToCloneMovableCollections' with 'comment'"); // Make shard0 the primary shard for both dbName0 and dbName1. assert.commandWorked(st.s.adminCommand({enableSharding: dbName0, primaryShard: st.shard0.shardName})); assert.commandWorked(st.s.adminCommand({enableSharding: dbName1, primaryShard: st.shard0.shardName})); // Make dbName0 have one collection n1, which is an unsharded collection. const ns0 = dbName0 + "." + collName0; createCollection(st, dbName0, collName0, trackUnshardedCollections); assert.commandWorked(st.s.getCollection(ns0).insert([{x: 1}])); // Make dbName1 have two collections ns1 and ns2, where ns1 is an unsharded collection and ns2 // is a sharded collection with [MinKey, 0] on shard0 and [0, MaxKey] on shard1. const ns1 = dbName1 + "." + collName0; createCollection(st, dbName1, collName0, trackUnshardedCollections); assert.commandWorked(st.s.getCollection(ns1).insert({x: 1})); const ns2 = dbName1 + "." + collName1; assert.commandWorked(st.s.adminCommand({shardCollection: ns2, key: {x: 1}})); assert.commandWorked(st.s.getCollection(ns2).insert([{x: -1}, {x: 1}])); assert.commandWorked(st.s.adminCommand({split: ns2, middle: {x: 0}})); assert.commandWorked( st.s.adminCommand({moveChunk: ns2, find: {x: 0}, to: st.shard1.shardName, _waitForDelete: true}), ); const comment = extractUUIDFromObject(UUID()); const movePrimaryFp0 = configureFailPoint(shard0Primary, "movePrimaryFailIfNeedToCloneMovableCollections", { comment, }); // The movePrimary command below should not fail whether not there is data for dbName0 to clone // since the command is run without the 'comment' above. const movePrimaryRes0 = st.s.adminCommand({movePrimary: dbName0, to: st.shard1.shardName}); assert.commandWorked(movePrimaryRes0); // The movePrimary command below should fail if there is data for dbName1 to clone since the // command is run with the 'comment' above. That is, it should fail if the unsharded // collection ns1 is untracked since movePrimary doesn't move tracked unsharded collections. const movePrimaryRes1 = st.s.adminCommand({movePrimary: dbName1, to: st.shard1.shardName, comment}); if (trackUnshardedCollections) { assert.commandWorked(movePrimaryRes1); } else { assert.commandFailedWithCode(movePrimaryRes1, 9046501); // Manually move the untracked unsharded collection ns1 to shard1. assert.commandWorked(st.s.adminCommand({moveCollection: ns1, toShard: st.shard1.shardName})); // The movePrimary command below should not fail since there is no data to clone. The // sharded collection n2 still has a chunk on shard0 but the collection is sharded so it // does not prevent shard1 from becoming the primary shard. assert.commandWorked(st.s.adminCommand({movePrimary: dbName1, to: st.shard1.shardName, comment})); } movePrimaryFp0.off(); const transitionRes0 = assert.commandWorked(st.s.adminCommand({transitionToDedicatedConfigServer: 1})); jsTest.log("The first transitionToDedicatedConfigServer response: " + tojson(transitionRes0)); if (trackUnshardedCollections) { assert.eq(transitionRes0.state, "started", transitionRes0); assert.eq(transitionRes0.dbsToMove.length, 0, transitionRes0); assert.eq(transitionRes0.collectionsToMove.length, 2, transitionRes0); assert.eq( transitionRes0.note, "you need to call moveCollection for collectionsToMove and afterwards movePrimary for the dbsToMove", transitionRes0, ); } else { assert.eq(transitionRes0.state, "started", transitionRes0); assert.eq(transitionRes0.dbsToMove.length, 0, transitionRes0); assert.eq(transitionRes0.collectionsToMove.length, 0, transitionRes0); assert(transitionRes0.note === undefined); } const transitionRes1 = assert.commandWorked(st.s.adminCommand({transitionToDedicatedConfigServer: 1})); jsTest.log("The second transitionToDedicatedConfigServer response: " + tojson(transitionRes1)); assert.eq(transitionRes1.state, "ongoing", transitionRes1); assert.eq(transitionRes1.remaining.dbs, 0, transitionRes1); // There is still data left on shard0. if (trackUnshardedCollections) { assert.gte(transitionRes1.remaining.chunks, 0, transitionRes1); assert.eq(transitionRes1.remaining.jumboChunks, 0, transitionRes1); assert.eq(transitionRes1.remaining.collectionsToMove, 2, transitionRes1); assert.eq(transitionRes1.collectionsToMove.length, 2, transitionRes1); assert.eq(transitionRes1.dbsToMove.length, 0, transitionRes1); assert.eq( transitionRes1.note, "you need to call moveCollection for collectionsToMove and afterwards movePrimary for the dbsToMove", transitionRes1, ); } else { assert.gte(transitionRes1.remaining.chunks, 0, transitionRes1); assert.eq(transitionRes1.remaining.jumboChunks, 0, transitionRes1); assert.eq(transitionRes1.remaining.collectionsToMove, 0, transitionRes1); assert.eq(transitionRes1.collectionsToMove.length, 0, transitionRes1); assert.eq(transitionRes1.dbsToMove.length, 0, transitionRes1); assert(transitionRes1.note === undefined); } // Move the remaining data on shard0 to shard1. assert.commandWorked( st.s.adminCommand({moveChunk: ns2, find: {x: -1}, to: st.shard1.shardName, _waitForDelete: true}), ); if (trackUnshardedCollections) { // movePrimary doesn't move tracked unsharded collections so n0 and n1 still need to moved // manually to shard1. assert.commandWorked(st.s.adminCommand({moveCollection: ns0, toShard: st.shard1.shardName})); assert.commandWorked(st.s.adminCommand({moveCollection: ns1, toShard: st.shard1.shardName})); } // Move the chunk(s) for config.system.sessions if needed. const collUuid = getCollectionUuid(st.s.getDB("config"), "system.sessions"); const chunkDocs = st.s.getCollection("config.chunks").find({uuid: collUuid, shard: "config"}).toArray(); for (let chunkDoc of chunkDocs) { assert.commandWorked( st.s.adminCommand({ moveChunk: "config.system.sessions", find: chunkDoc.min, to: st.shard1.shardName, _waitForDelete: true, }), ); } // The command is compleated when all the orphans are deleted and no range deletion tasks are // left. Every migration above runs with _waitForDelete=true, which guarantees every range // deletions task to be processed but not necessarly removed from disk yet. assert.soon( () => { const transitionRes2 = assert.commandWorked(st.s.adminCommand({transitionToDedicatedConfigServer: 1})); jsTest.log("The third transitionToDedicatedConfigServer response: " + tojson(transitionRes2)); return transitionRes2.state == "completed"; }, "transitionToDedicatedConfigServer did not return 'complete' status within the timeout.", 60000 /*1 minute*/, ); jsTest.log("Testing 'movePrimaryFailIfNeedToCloneMovableCollections' without 'comment'"); // Make shard1 the primary shard for dbName2. assert.commandWorked(st.s.adminCommand({enableSharding: dbName2, primaryShard: st.shard1.shardName})); // Make dbName2 have one collection n3, which is an unsharded collection. const ns3 = dbName2 + "." + collName0; createCollection(st, dbName2, collName0, trackUnshardedCollections); assert.commandWorked(st.s.getCollection(ns3).insert([{x: 1}])); const movePrimaryFp1 = configureFailPoint(shard1Primary, "movePrimaryFailIfNeedToCloneMovableCollections"); // The movePrimary command below should fail if there is data for dbName2 to clone. That is, it // should fail if the unsharded collection ns3 is untracked since movePrimary doesn't move // tracked unsharded collections. const movePrimaryRes2 = st.s.adminCommand({movePrimary: dbName2, to: st.shard2.shardName}); if (trackUnshardedCollections) { assert.commandWorked(movePrimaryRes2); } else { assert.commandFailedWithCode(movePrimaryRes2, 9046501); } movePrimaryFp1.off(); st.stop(); } function testInternalCollections({featureFlagReshardingForTimeseries}) { jsTest.log("Running tests for internal collections " + tojson({featureFlagReshardingForTimeseries})); const st = new ShardingTest({ shards: 2, configShard: true, rs: {setParameter: {featureFlagReshardingForTimeseries}}, }); const shard0Primary = st.rs0.getPrimary(); const collName = "testColl"; const viewName = "testCollView"; const movePrimaryFp = configureFailPoint(shard0Primary, "movePrimaryFailIfNeedToCloneMovableCollections"); jsTest.log("Testing a database with a view"); const dbName0 = "testDbWithView"; const testDB0 = st.s.getDB(dbName0); // Make shard0 the primary shard for dbName0. assert.commandWorked(st.s.adminCommand({enableSharding: dbName0, primaryShard: st.shard0.shardName})); // Create a test collection and a view on it. assert.commandWorked(testDB0.getCollection(collName).insert({x: 1})); assert.commandWorked(testDB0.runCommand({create: viewName, viewOn: collName, pipeline: [{$match: {}}]})); assert.commandFailedWithCode(st.s.adminCommand({movePrimary: dbName0, to: st.shard1.shardName}), 9046501); assert.commandWorked(st.s.adminCommand({moveCollection: dbName0 + "." + collName, toShard: st.shard1.shardName})); assert.commandFailedWithCode( st.s.adminCommand({moveCollection: dbName0 + ".system.views", toShard: st.shard1.shardName}), // Tracking of a system.views collection is not supported. ErrorCodes.IllegalOperation, ); assert.commandFailedWithCode( st.s.adminCommand({moveCollection: dbName0 + "." + viewName, toShard: st.shard1.shardName}), // Tracking of a view is not supported. ErrorCodes.NamespaceNotFound, ); // movePrimary should be allowed to move the system.views collection and the view. assert.commandWorked(st.s.adminCommand({movePrimary: dbName0, to: st.shard1.shardName})); if (buildInfo().modules.includes("enterprise")) { jsTest.log("Testing a database with an FLE collection"); const dbName1 = "testDbWithFLE"; // Make shard0 the primary shard for dbName1. assert.commandWorked(st.s.adminCommand({enableSharding: dbName1, primaryShard: st.shard0.shardName})); // Create a test collection with FLE enabled. const encryptedClient = new EncryptedClient(st.s, dbName1); assert.commandWorked( encryptedClient.createEncryptionCollection(collName, { encryptedFields: { "fields": [{"path": "x", "bsonType": "int", "queries": {"queryType": "equality"}}], }, }), ); assert.commandFailedWithCode(st.s.adminCommand({movePrimary: dbName1, to: st.shard1.shardName}), 9046501); assert.commandWorked( st.s.adminCommand({moveCollection: dbName1 + "." + collName, toShard: st.shard1.shardName}), ); // movePrimary should be allowed to move the FLE internal collections but not the keystore // collection. assert.commandFailedWithCode(st.s.adminCommand({movePrimary: dbName1, to: st.shard1.shardName}), 9046501); assert.commandWorked(st.s.adminCommand({moveCollection: dbName1 + ".keystore", toShard: st.shard1.shardName})); assert.commandWorked(st.s.adminCommand({movePrimary: dbName1, to: st.shard1.shardName})); } jsTest.log("Testing a database with a timeseries collection"); const dbName2 = "testDbWithTimeSeries"; const testDB2 = st.s.getDB(dbName2); // Make shard0 the primary shard for dbName2. assert.commandWorked(st.s.adminCommand({enableSharding: dbName2, primaryShard: st.shard0.shardName})); // Create a test timeseries collection. assert.commandWorked(testDB2.runCommand({create: collName, timeseries: {timeField: "x", metaField: "y"}})); if (featureFlagReshardingForTimeseries) { // If this feature flag is enabled, moveCollection should be allowed to move the timeseries // bucket collection but movePrimary shouldn't be allowed to. assert.commandFailedWithCode(st.s.adminCommand({movePrimary: dbName2, to: st.shard1.shardName}), 9046501); assert.commandWorked( st.s.adminCommand({moveCollection: dbName2 + "." + collName, toShard: st.shard1.shardName}), ); assert.commandWorked(st.s.adminCommand({movePrimary: dbName2, to: st.shard1.shardName})); } else { // If this feature flag is disabled, moveCollection should not be allowed to move the // timeseries bucket collection but movePrimary should be allowed to. assert.commandFailedWithCode( st.s.adminCommand({moveCollection: dbName2 + "." + collName, toShard: st.shard1.shardName}), ErrorCodes.IllegalOperation, ); assert.commandWorked(st.s.adminCommand({movePrimary: dbName2, to: st.shard1.shardName})); } jsTest.log("Testing a database with a system.js collection"); const dbName3 = "testDbWithSystemJS"; const testDB3 = st.s.getDB(dbName3); // Make shard0 the primary shard for dbName3. assert.commandWorked(st.s.adminCommand({enableSharding: dbName3, primaryShard: st.shard0.shardName})); assert.commandWorked( testDB3.getCollection("system.js").insert({ _id: "addOne", value: function (x) { return x + 1; }, }), ); assert.commandFailedWithCode( st.s.adminCommand({moveCollection: dbName3 + ".system.js", toShard: st.shard1.shardName}), // Tracking of a system.js collection is not supported. ErrorCodes.IllegalOperation, ); // movePrimary should be allowed to move the system.js collection. assert.commandWorked(st.s.adminCommand({movePrimary: dbName3, to: st.shard1.shardName})); jsTest.log("Testing a database with system.resharding. collection"); const dbName4 = "testDbWithSystemResharding"; const testDB4 = st.s.getDB(dbName4); // Make shard0 the primary shard for dbName4. assert.commandWorked(st.s.adminCommand({enableSharding: dbName4, primaryShard: st.shard0.shardName})); assert.commandWorked(testDB4.runCommand({create: "system.resharding.foo"})); // moveCollection does not have a way to tell that this is not a real resharding temporary // collection. However, movePrimary should not be allowed to move it. assert.commandFailedWithCode( st.s.adminCommand({moveCollection: dbName4 + ".system.resharding.foo", toShard: st.shard1.shardName}), // Can't move an internal resharding collection. ErrorCodes.IllegalOperation, ); assert.commandFailedWithCode(st.s.adminCommand({movePrimary: dbName4, to: st.shard1.shardName}), 9046501); movePrimaryFp.off(); st.stop(); } testUserCollections({trackUnshardedCollections: false}); testUserCollections({trackUnshardedCollections: true}); testInternalCollections({featureFlagReshardingForTimeseries: false}); testInternalCollections({featureFlagReshardingForTimeseries: true});