SERVER-83289 Added rewriteCollection command to handle resharding on the existing shard key (#43342)

GitOrigin-RevId: c799cd1626308c80d122243cdd4dbd046e470805
This commit is contained in:
natalie-hill 2025-11-05 11:06:11 -05:00 committed by MongoDB Bot
parent 9493ebca54
commit 305538e0cb
42 changed files with 727 additions and 6 deletions

5
.github/CODEOWNERS vendored
View File

@ -865,6 +865,9 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
# The following patterns are parsed from ./jstests/core_sharding/global_catalog/OWNERS.yml
/jstests/core_sharding/global_catalog/**/* @10gen/server-catalog-and-routing-ddl @svc-auto-approve-bot
# The following patterns are parsed from ./jstests/core_sharding/resharding/OWNERS.yml
/jstests/core_sharding/resharding/**/* @10gen/server-cluster-scalability @svc-auto-approve-bot
# The following patterns are parsed from ./jstests/core_sharding/sharded_cluster_topology/OWNERS.yml
/jstests/core_sharding/sharded_cluster_topology/**/* @10gen/server-catalog-and-routing-routing-and-topology @svc-auto-approve-bot
@ -1545,6 +1548,7 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
/jstests/sharding/**/*range*deleter* @10gen/server-cluster-scalability @svc-auto-approve-bot
/jstests/sharding/**/*range*deletion* @10gen/server-cluster-scalability @svc-auto-approve-bot
/jstests/sharding/**/*retryable_write* @10gen/server-transactions @svc-auto-approve-bot
/jstests/sharding/**/*rewrite_collection* @10gen/server-cluster-scalability @svc-auto-approve-bot
/jstests/sharding/**/*transaction* @10gen/server-transactions @svc-auto-approve-bot
/jstests/sharding/**/*txn* @10gen/server-transactions @svc-auto-approve-bot
/jstests/sharding/**/*_with_id_without_shard_key* @10gen/server-cluster-scalability @svc-auto-approve-bot
@ -2057,6 +2061,7 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
/src/mongo/db/global_catalog/ddl/**/*move*collection* @10gen/server-cluster-scalability @svc-auto-approve-bot
/src/mongo/db/global_catalog/ddl/**/*refine_collection_shard_key* @10gen/server-cluster-scalability @svc-auto-approve-bot
/src/mongo/db/global_catalog/ddl/**/*unshard*collection* @10gen/server-cluster-scalability @svc-auto-approve-bot
/src/mongo/db/global_catalog/ddl/**/*rewrite*collection* @10gen/server-cluster-scalability @svc-auto-approve-bot
# The following patterns are parsed from ./src/mongo/db/global_catalog/router_role_api/OWNERS.yml
/src/mongo/db/global_catalog/router_role_api/**/* @10gen/server-catalog-and-routing-routing-and-topology @svc-auto-approve-bot

View File

@ -477,6 +477,19 @@ export const authCommandsLib = {
},
],
},
{
testname: "abortRewriteCollection",
command: {abortRewriteCollection: "test.x"},
skipUnlessSharded: true,
testcases: [
{
runOnDb: adminDbName,
roles: Object.extend({enableSharding: 1}, roles_clusterManager),
privileges: [{resource: {db: "test", collection: "x"}, actions: ["rewriteCollection"]}],
expectFail: true,
},
],
},
{
testname: "abortUnshardCollection",
command: {abortUnshardCollection: "test.x"},
@ -6851,7 +6864,21 @@ export const authCommandsLib = {
{runOnDb: secondDbName, roles: {}},
],
},
{
testname: "rewriteCollection",
command: {rewriteCollection: "test.x"},
skipUnlessSharded: true,
testcases: [
{
runOnDb: adminDbName,
roles: Object.extend({enableSharding: 1}, roles_clusterManager),
privileges: [{resource: {db: "test", collection: "x"}, actions: ["rewriteCollection"]}],
expectFail: true,
},
{runOnDb: firstDbName, roles: {}},
{runOnDb: secondDbName, roles: {}},
],
},
{
testname: "_configsvrReshardCollection",
command: {_configsvrReshardCollection: "test.x", key: {_id: 1}},

View File

@ -69,6 +69,16 @@ function reshardCollectionCmd() {
);
}
function rewriteCollectionCmd() {
assert.commandWorked(
mongos.adminCommand({
rewriteCollection: kNsName,
numInitialChunks: 1,
zones: [{zone: zoneName, min: {_id: MinKey}, max: {_id: MaxKey}}],
}),
);
}
function moveCollectionCmd() {
assert.commandWorked(mongos.adminCommand({moveCollection: kNsName, toShard: nonPrimaryShard}));
}
@ -163,6 +173,33 @@ for (let watchCollectionParameter of [kCollName, 1]) {
},
);
jsTest.log(`Validate behavior of rewriteCollection against a change stream watching at '${watchLevel}' level`);
validateExpectedEventAndConfirmResumability(
prepareShardedCollection,
rewriteCollectionCmd,
watchCollectionParameter,
{
"operationType": "reshardCollection",
"collectionUUID": 0,
"ns": {"db": "reshard_collection_event", "coll": "coll"},
"operationDescription": {
"reshardUUID": 0,
"shardKey": {"_id": 1},
"oldShardKey": {"_id": 1},
"unique": false,
"numInitialChunks": NumberLong(1),
"provenance": "rewriteCollection",
"zones": [
{
"zone": "zone1",
"min": {"_id": {"$minKey": 1}},
"max": {"_id": {"$maxKey": 1}},
},
],
},
},
);
jsTest.log(`Validate behavior of moveCollection against a change stream watching at '${watchLevel}' level`);
validateExpectedEventAndConfirmResumability(
prepareUnshardedCollection,

View File

@ -233,6 +233,7 @@ let viewsCommandTests = {
streams_updateConnection: {skip: isAnInternalCommand},
_transferMods: {skip: isAnInternalCommand},
abortMoveCollection: {skip: isUnrelated},
abortRewriteCollection: {skip: isUnrelated},
abortReshardCollection: {skip: isUnrelated},
abortTransaction: {skip: isUnrelated},
abortUnshardCollection: {skip: isUnrelated},
@ -680,6 +681,13 @@ let viewsCommandTests = {
expectFailure: true,
isAdminCommand: true,
},
rewriteCollection: {
command: {rewriteCollection: "test.view"},
expectedErrorCode: [ErrorCodes.NamespaceNotSharded, ErrorCodes.NamespaceNotFound],
skipStandalone: true,
expectFailure: true,
isAdminCommand: true,
},
revokePrivilegesFromRole: {
command: {
revokePrivilegesFromRole: "testrole",

View File

@ -0,0 +1,5 @@
version: 1.0.0
filters:
- "*":
approvers:
- 10gen/server-cluster-scalability

View File

@ -75,6 +75,18 @@ function runReshardCollection(host, ns, key, performVerification, countDownLatch
return res;
}
function runRewriteCollection(host, ns, performVerification, countDownLatch) {
const mongos = new Mongo(host);
const cmdObj = {rewriteCollection: ns};
if (performVerification !== null) {
cmdObj.performVerification = performVerification;
}
const res = mongos.adminCommand(cmdObj);
countDownLatch.countDown();
return res;
}
function runUnshardCollection(host, ns, toShard, performVerification, countDownLatch) {
const mongos = new Mongo(host);
const cmdObj = {unshardCollection: ns, toShard};
@ -204,6 +216,41 @@ function testReshardCollection(performVerification) {
testResharding(reshardThread, reshardCountDownLatch, ns, performVerification);
}
function testRewriteCollection(performVerification) {
if (MongoRunner.compareBinVersions(jsTestOptions().mongosBinVersion, "8.3") < 0) {
// rewriteCollection is not supported in versions prior to 8.3, skip
return;
}
const collName = getTestCollectionName();
const ns = dbName + "." + collName;
const numDocs = 1000;
const docs = makeDocuments(numDocs);
assert.commandWorked(testDB.getCollection(collName).insert(docs));
assert.commandWorked(db.adminCommand({shardCollection: ns, key: {_id: 1}}));
assert.commandWorked(db.adminCommand({split: ns, middle: {_id: 0}}));
assert.commandWorked(
db.adminCommand({
moveChunk: ns,
find: {_id: 0},
to: getNonOwningShardName(dbName, collName),
_waitForDelete: true,
}),
);
jsTest.log("Testing rewriteCollection with " + tojson({performVerification}));
const rewriteCountDownLatch = new CountDownLatch(1);
const rewriteThread = new Thread(
runRewriteCollection,
db.getMongo().host,
ns,
performVerification,
rewriteCountDownLatch,
);
testResharding(rewriteThread, rewriteCountDownLatch, ns, performVerification);
}
function testUnshardCollection(performVerification) {
const collName = getTestCollectionName();
const ns = dbName + "." + collName;
@ -249,6 +296,7 @@ function testMoveCollection(performVerification) {
function runTest(performVerification) {
testReshardCollection(performVerification);
testRewriteCollection(performVerification);
testUnshardCollection(performVerification);
testMoveCollection(performVerification);
}

View File

@ -0,0 +1,71 @@
/**
* Tests for basic functionality of the rewriteCollection command.
*
* @tags: [
* requires_fcv_83,
* assumes_balancer_off,
* # Stepdown test coverage is already provided by the resharding FSM suites.
* does_not_support_stepdowns,
* # This test performs explicit calls to shardCollection
* assumes_unsharded_collection,
* ]
*/
import {ReshardCollectionCmdTest} from "jstests/sharding/libs/reshard_collection_util.js";
import {getShardNames} from "jstests/sharding/libs/sharding_util.js";
const shardNames = getShardNames(db);
const collName = jsTestName();
const dbName = db.getName();
const ns = dbName + "." + collName;
const mongos = db.getMongo();
const numInitialDocs = 500;
const zoneNames = ["z0", "z1", "z3"];
if (shardNames.length < 2) {
jsTest.log.info(jsTestName() + " will not run; at least 2 shards are required.");
quit();
}
const reshardCmdTest = new ReshardCollectionCmdTest({
mongos,
dbName,
collName,
numInitialDocs,
skipDirectShardChecks: true,
});
jsTest.log.info("Succeed performing basic rewriteCollection command");
reshardCmdTest.assertReshardCollOk({rewriteCollection: ns, numInitialChunks: 2}, 2);
let additionalSetup = function (test) {
const ns = test._ns;
assert.commandWorked(mongos.adminCommand({addShardToZone: shardNames[0], zone: zoneNames[0]}));
assert.commandWorked(mongos.adminCommand({addShardToZone: shardNames[1], zone: zoneNames[1]}));
};
jsTest.log.info("Succeed when rewriting all data to one zone/shard");
reshardCmdTest.assertReshardCollOk(
{
rewriteCollection: ns,
numInitialChunks: 1,
zones: [{zone: zoneNames[0], min: {oldKey: MinKey}, max: {oldKey: MaxKey}}],
},
1,
[{recipientShardId: shardNames[0], min: {oldKey: MinKey}, max: {oldKey: MaxKey}}],
[{zone: zoneNames[0], min: {oldKey: MinKey}, max: {oldKey: MaxKey}}],
additionalSetup,
);
jsTest.log.info("Fail when a zone with no shards is provided");
assert.throwsWithCode(
() =>
reshardCmdTest.assertReshardCollOk(
{
rewriteCollection: ns,
zones: [{zone: zoneNames[2], min: {oldKey: MinKey}, max: {oldKey: MaxKey}}],
},
1,
),
[ErrorCodes.BadValue, ErrorCodes.ZoneNotFound],
);

View File

@ -142,6 +142,7 @@ function runCommandWithRetryUponMigration(conn, dbName, commandName, commandObj,
"createIndexes",
"moveCollection",
"reshardCollection",
"rewriteCollection",
"unshardCollection",
]);

View File

@ -1,5 +1,5 @@
/**
* Overrides runCommand to retry "reshardCollection", "moveCollection", and "unshardCollection"
* Overrides runCommand to retry "reshardCollection", "rewriteCollection", "moveCollection", and "unshardCollection"
* commands if they fail with OplogQueryMinTsMissing or SnapshotUnavailable.
*/
@ -9,7 +9,7 @@ import {OverrideHelpers} from "jstests/libs/override_methods/override_helpers.js
const kTimeout = 20 * 60 * 1000;
const kInterval = 100;
const kRetryableCommands = ["reshardCollection", "moveCollection", "unshardCollection"];
const kRetryableCommands = ["reshardCollection", "rewriteCollection", "moveCollection", "unshardCollection"];
const kRetryableErrorCodes = [ErrorCodes.OplogQueryMinTsMissing, ErrorCodes.SnapshotUnavailable];

View File

@ -221,6 +221,7 @@ const wcCommandsTests = {
_transferMods: {skip: "internal command"},
abortMoveCollection: {skip: "does not accept write concern"},
abortReshardCollection: {skip: "does not accept write concern"},
abortRewriteCollection: {skip: "does not accept write concern"},
abortTransaction: {
success: {
// Basic abort transaction
@ -2246,6 +2247,7 @@ const wcCommandsTests = {
replSetUpdatePosition: {skip: "does not accept write concern"},
resetPlacementHistory: {skip: "internal command"},
reshardCollection: {skip: "does not accept write concern"},
rewriteCollection: {skip: "does not accept write concern"},
revokePrivilegesFromRole: {
targetConfigServer: true,
noop: {
@ -3430,6 +3432,7 @@ const wcTimeseriesViewsCommandsTests = {
_transferMods: {skip: "internal command"},
abortMoveCollection: {skip: "does not accept write concern"},
abortReshardCollection: {skip: "does not accept write concern"},
abortRewriteCollection: {skip: "does not accept write concern"},
abortTransaction: {skip: "not supported on timeseries views"},
abortUnshardCollection: {skip: "does not accept write concern"},
addShard: {skip: "unrelated"},
@ -4347,6 +4350,7 @@ const wcTimeseriesViewsCommandsTests = {
revokePrivilegesFromRole: wcCommandsTests["revokePrivilegesFromRole"],
revokeRolesFromRole: wcCommandsTests["revokeRolesFromRole"],
revokeRolesFromUser: wcCommandsTests["revokeRolesFromUser"],
rewriteCollection: {skip: "does not accept write concern"},
rolesInfo: {skip: "does not accept write concern"},
rotateCertificates: {skip: "does not accept write concern"},
rotateFTDC: {skip: "does not accept write concern"},

View File

@ -67,6 +67,22 @@ assert.commandWorked(
}),
);
// Test rewriteCollection with OplogQueryMinTsMissing. The command should be retried.
failNextCommandWithCode(testDB, "rewriteCollection", ErrorCodes.OplogQueryMinTsMissing);
assert.commandWorked(
st.s.adminCommand({
rewriteCollection: ns,
}),
);
// Test rewriteCollection with SnapshotUnavailable. The command should be retried.
failNextCommandWithCode(testDB, "rewriteCollection", ErrorCodes.SnapshotUnavailable);
assert.commandWorked(
st.s.adminCommand({
rewriteCollection: ns,
}),
);
// Test moveCollection with OplogQueryMinTsMissing. The command should be retried.
const unshardedCollName = "unshardedColl";
const unshardedNs = dbName + "." + unshardedCollName;
@ -133,6 +149,15 @@ assert.commandFailedWithCode(
ErrorCodes.InternalError,
);
// Test rewriteCollection with a non-retryable error.
failNextCommandWithCode(testDB, "rewriteCollection", ErrorCodes.InternalError);
assert.commandFailedWithCode(
st.s.adminCommand({
rewriteCollection: ns,
}),
ErrorCodes.InternalError,
);
// Test moveCollection with a non-retryable error.
failNextCommandWithCode(testDB, "moveCollection", ErrorCodes.InternalError);
assert.commandFailedWithCode(

View File

@ -42,6 +42,13 @@ function runTest() {
ErrorCodes.IllegalOperation,
);
assert.commandFailedWithCode(
st.s.adminCommand({
rewriteCollection: kFullName,
}),
ErrorCodes.IllegalOperation,
);
assert.commandFailedWithCode(
st.s.adminCommand({unshardCollection: kFullName, toShard: kOther}),
ErrorCodes.IllegalOperation,

View File

@ -179,6 +179,10 @@ const allCommands = {
// Skipping command because it requires testing through a parallel shell.
skip: requiresParallelShell,
},
abortRewriteCollection: {
// Skipping command because it requires testing through a parallel shell.
skip: requiresParallelShell,
},
abortTransaction: {
doesNotRunOnStandalone: true,
fullScenario: function (conn) {
@ -1358,6 +1362,7 @@ const allCommands = {
skip: "Cannot run while downgrading",
},
reshardCollection: {skip: cannotRunWhileDowngrading},
rewriteCollection: {skip: cannotRunWhileDowngrading},
revokePrivilegesFromRole: {
setUp: function (conn) {
assert.commandWorked(conn.getDB(dbName).runCommand({create: collName}));

View File

@ -165,6 +165,7 @@ const allCommands = {
_transferMods: {skip: isPrimaryOnly},
abortMoveCollection: {skip: isPrimaryOnly},
abortReshardCollection: {skip: isPrimaryOnly},
abortRewriteCollection: {skip: isPrimaryOnly},
abortTransaction: {skip: isPrimaryOnly},
abortUnshardCollection: {skip: isPrimaryOnly},
aggregate: {

View File

@ -37,6 +37,9 @@ filters:
- "*retryable_write*":
approvers:
- 10gen/server-transactions
- "*rewrite_collection*":
approvers:
- 10gen/server-cluster-scalability
- "*transaction*":
approvers:
- 10gen/server-transactions

View File

@ -0,0 +1,73 @@
/**
* Tests for basic functionality of the abort rewrite collection feature.
*
* @tags: [
* requires_fcv_83,
* ]
*/
import {configureFailPoint} from "jstests/libs/fail_point_util.js";
import {funWithArgs} from "jstests/libs/parallel_shell_helpers.js";
import {ShardingTest} from "jstests/libs/shardingtest.js";
let st = new ShardingTest({mongos: 1, shards: 2});
const dbName = "test";
const collName = "foo";
const ns = dbName + "." + collName;
let mongos = st.s0;
let shard0 = st.shard0.shardName;
let shard1 = st.shard1.shardName;
let shardKeyMin = -500;
let shardKeyMax = 500;
assert.commandWorked(st.s.adminCommand({enableSharding: dbName, primaryShard: shard0}));
const coll = mongos.getDB(dbName)[collName];
for (let i = shardKeyMin; i < shardKeyMax; ++i) {
assert.commandWorked(coll.insert({_id: i}));
}
mongos.getDB(dbName).adminCommand({shardCollection: ns, key: {_id: 1}});
let failpoint = configureFailPoint(st.rs1.getPrimary(), "reshardingPauseRecipientDuringCloning");
const awaitResult = startParallelShell(
funWithArgs(function (ns) {
assert.commandFailedWithCode(db.adminCommand({rewriteCollection: ns}), ErrorCodes.ReshardCollectionAborted);
}, ns),
st.s.port,
);
failpoint.wait();
// Verify that the provenance field is appended to the currentOp
const filter = {
type: "op",
"originatingCommand.reshardCollection": ns,
"provenance": "rewriteCollection",
};
assert.soon(() => {
return (
st.s
.getDB("admin")
.aggregate([{$currentOp: {allUsers: true, localOps: false}}, {$match: filter}])
.toArray().length >= 1
);
});
assert.commandWorked(mongos.adminCommand({abortRewriteCollection: ns}));
failpoint.off();
awaitResult();
// Confirm that the operation started and was cancelled.
const metrics = st.config0.getDB("admin").serverStatus({}).shardingStatistics.rewriteCollection;
assert.eq(metrics.countStarted, 1);
assert.eq(metrics.countSucceeded, 0);
assert.eq(metrics.countFailed, 0);
assert.eq(metrics.countCanceled, 1);
st.stop();

View File

@ -175,6 +175,10 @@ const allCommands = {
// Skipping command because it requires testing through a parallel shell.
skip: requiresParallelShell,
},
abortRewriteCollection: {
// Skipping command because it requires testing through a parallel shell.
skip: requiresParallelShell,
},
abortTransaction: {skip: "Requires changes to permissions or number of shards"},
abortUnshardCollection: {
// Skipping command because it requires testing through a parallel shell.
@ -1045,6 +1049,7 @@ const allCommands = {
},
resetPlacementHistory: {skip: requiresMongoS},
reshardCollection: {skip: requiresMongoS},
rewriteCollection: {skip: requiresMongoS},
revokePrivilegesFromRole: {
setUp: function (mongoS, withDirectConnections) {
assert.commandWorked(withDirectConnections.getDB(dbName).runCommand({create: collName}));

View File

@ -137,6 +137,39 @@ function testReshardCollectionQuerySamplingEnabled(st) {
assertQuerySampling(dbName, collName, true /* isActive */, st);
}
function testRewriteCollectionQuerySamplingEnabled(st) {
if (MongoRunner.compareBinVersions(jsTestOptions().mongosBinVersion, "8.3") < 0) {
// rewriteCollection is not supported in versions prior to 8.3, skip
return;
}
assert(st);
const {dbName, collName} = setUpCollection(st, true /* isShardedColl */);
const ns = dbName + "." + collName;
const srcShard = st.shard0.shardName;
const dstShard = st.shard1.shardName;
const conn = st.s;
jsTest.log(`Testing rewriteCollection ${tojson({dbName, collName, srcShard, dstShard})}`);
enableQuerySampling(st, dbName, collName);
validateQueryAnalyzerDoc(conn, dbName, collName);
// Insert enough documents with field "x" into sharded collection to meet cardinality requirement for resharding.
for (let i = 0; i < 1000; i++) {
st.s.getCollection(ns).insert({x: i});
}
// Reshard the sharded collection on the same shard key.
assert.commandWorked(st.s.adminCommand({rewriteCollection: ns}));
validateQueryAnalyzerDoc(conn, dbName, collName);
assertQuerySampling(dbName, collName, true /* isActive */, st);
}
function testUnshardCollectionQuerySamplingEnabled(st) {
assert(st);
@ -215,6 +248,7 @@ const mongosSetParametersOpts = {
testMoveCollectionQuerySamplingEnabled(st);
testUnshardCollectionQuerySamplingEnabled(st);
testReshardCollectionQuerySamplingEnabled(st);
testRewriteCollectionQuerySamplingEnabled(st);
for (let isShardedColl of [true, false]) {
testReshardQuerySamplingDisabled(st, isShardedColl);

View File

@ -310,6 +310,7 @@ const allTestCases = {
_dropMirrorMaestroConnections: {skip: "not on a user database", conditional: true},
abortMoveCollection: {skip: "always targets the config server"},
abortReshardCollection: {skip: "always targets the config server"},
abortRewriteCollection: {skip: "always targets the config server"},
abortTransaction: {skip: "unversioned and uses special targetting rules"},
abortUnshardCollection: {skip: "always targets the config server"},
addShard: {skip: "not on a user database"},
@ -778,6 +779,7 @@ const allTestCases = {
revokePrivilegesFromRole: {skip: "always targets the config server"},
revokeRolesFromRole: {skip: "always targets the config server"},
revokeRolesFromUser: {skip: "always targets the config server"},
rewriteCollection: {skip: "requires sharded collection"},
rolesInfo: {skip: "always targets the config server"},
rotateCertificates: {skip: "executes locally on mongos (not sent to any remote node)"},
rotateFTDC: {skip: "executes locally on mongos (not sent to any remote node)"},

View File

@ -26,4 +26,6 @@ export const commandsAddedToMongosSinceLastLTS = [
"stopTransitionToDedicatedConfigServer",
"commitShardRemoval",
"commitTransitionToDedicatedConfigServer",
"rewriteCollection",
"abortRewriteCollection",
];

View File

@ -128,6 +128,7 @@ export let MongosAPIParametersUtil = (function () {
{commandName: "_mongotConnPoolStats", skip: "internal API"},
{commandName: "abortMoveCollection", skip: "TODO(SERVER-108802)"},
{commandName: "abortReshardCollection", skip: "TODO(SERVER-108802)"},
{commandName: "abortRewriteCollection", skip: "TODO(SERVER-108802)"},
{commandName: "abortUnshardCollection", skip: "TODO(SERVER-108802)"},
{commandName: "analyze", skip: "TODO(SERVER-108802)"},
{
@ -1469,6 +1470,19 @@ export let MongosAPIParametersUtil = (function () {
command: () => ({reshardCollection: "db.collection", key: {_id: 1}}),
},
},
{
commandName: "rewriteCollection",
run: {
inAPIVersion1: false,
permittedInTxn: false,
shardCommandName: "_shardsvrReshardCollection",
requiresShardedCollection: true,
// rewriteCollection calls reshardCollection, which internally does atClusterTime reads.
requiresCommittedReads: true,
runsAgainstAdminDb: true,
command: () => ({rewriteCollection: "db.collection"}),
},
},
{
commandName: "revokePrivilegesFromRole",
run: {

View File

@ -306,7 +306,15 @@ export class ReshardCollectionCmdTest {
const endTime = Date.now();
this._reshardDuration = (endTime - startTime) / 1000;
this._verifyShardKey(commandObj.key);
let commandShardKey = commandObj.key;
if ("rewriteCollection" in commandObj) {
// If this is a rewriteCollection command then it doesn't include a shard key, so we need to use the existing shard key for all checks.
const db = this._mongos.getDB(this._dbName);
const coll = this._timeseries ? getTimeseriesCollForDDLOps(db, db[this._collName]) : db[this._collName];
commandShardKey = coll.getShardKey();
} else {
this._verifyShardKey(commandShardKey);
}
if (expectedChunks) {
this._verifyTemporaryReshardingCollectionExistsWithCorrectOptions(
@ -315,11 +323,11 @@ export class ReshardCollectionCmdTest {
);
}
this._verifyTagsDocumentsAfterOperationCompletes(this._ns, Object.keys(commandObj.key), expectedZones);
this._verifyTagsDocumentsAfterOperationCompletes(this._ns, Object.keys(commandShardKey), expectedZones);
this._verifyChunksMatchExpected(expectedChunkNum, expectedChunks);
this._verifyIndexesCreated(indexes, commandObj.key);
this._verifyIndexesCreated(indexes, commandShardKey);
this._verifyDocumentsExist(docs, collection);

View File

@ -502,6 +502,9 @@ export var ReshardingTest = class {
_presetReshardedChunks: newChunks,
};
break;
case "rewriteCollection":
command = {rewriteCollection: ns};
break;
case "moveCollection":
command = {moveCollection: ns};
break;

View File

@ -225,6 +225,7 @@ let testCases = {
_transferMods: {skip: "internal command"},
abortMoveCollection: {skip: "does not accept read or write concern"},
abortReshardCollection: {skip: "does not accept read or write concern"},
abortRewriteCollection: {skip: "does not accept read or write concern"},
abortTransaction: {
setUp: function (conn) {
assert.commandWorked(conn.getDB(db).runCommand({create: coll, writeConcern: {w: "majority"}}));
@ -711,6 +712,7 @@ let testCases = {
replSetUpdatePosition: {skip: "does not accept read or write concern"},
resetPlacementHistory: {skip: "does not accept read or write concern"},
reshardCollection: {skip: "does not accept read or write concern"},
rewriteCollection: {skip: "does not accept read or write concern"},
resync: {skip: "does not accept read or write concern"},
revokePrivilegesFromRole: {
setUp: function (conn) {

View File

@ -57,6 +57,7 @@ import {ShardingTest} from "jstests/libs/shardingtest.js";
assert.commandFailedWithCode(mongos.adminCommand({abortUnshardCollection: ns}), ErrorCodes.IllegalOperation);
assert.commandFailedWithCode(mongos.adminCommand({abortMoveCollection: ns}), ErrorCodes.IllegalOperation);
assert.commandFailedWithCode(mongos.adminCommand({abortRewriteCollection: ns}), ErrorCodes.IllegalOperation);
assert.commandWorked(mongos.adminCommand({abortReshardCollection: ns}));

View File

@ -105,6 +105,7 @@ let testCases = {
_transferMods: {skip: "primary only"},
abortMoveCollection: {skip: "primary only"},
abortReshardCollection: {skip: "primary only"},
abortRewriteCollection: {skip: "primary only"},
abortTransaction: {skip: "primary only"},
abortUnshardCollection: {skip: "primary only"},
addShard: {skip: "primary only"},
@ -362,6 +363,7 @@ let testCases = {
revokePrivilegesFromRole: {skip: "primary only"},
revokeRolesFromRole: {skip: "primary only"},
revokeRolesFromUser: {skip: "primary only"},
rewriteCollection: {skip: "primary only"},
rolesInfo: {skip: "primary only"},
rotateCertificates: {skip: "does not return user data"},
rotateFTDC: {skip: "does not return user data"},

View File

@ -120,6 +120,7 @@ let testCases = {
_transferMods: {skip: "primary only"},
abortMoveCollection: {skip: "primary only"},
abortReshardCollection: {skip: "primary only"},
abortRewriteCollection: {skip: "primary only"},
abortTransaction: {skip: "primary only"},
abortUnshardCollection: {skip: "primary only"},
addShard: {skip: "primary only"},
@ -462,6 +463,7 @@ let testCases = {
revokePrivilegesFromRole: {skip: "primary only"},
revokeRolesFromRole: {skip: "primary only"},
revokeRolesFromUser: {skip: "primary only"},
rewriteCollection: {skip: "primary only"},
rolesInfo: {skip: "primary only"},
rotateCertificates: {skip: "does not return user data"},
rotateFTDC: {skip: "does not return user data"},

View File

@ -112,6 +112,7 @@ let testCases = {
_transferMods: {skip: "primary only"},
abortMoveCollection: {skip: "primary only"},
abortReshardCollection: {skip: "primary only"},
abortRewriteCollection: {skip: "primary only"},
abortTransaction: {skip: "primary only"},
abortUnshardCollection: {skip: "primary only"},
addShard: {skip: "primary only"},
@ -378,6 +379,7 @@ let testCases = {
revokePrivilegesFromRole: {skip: "primary only"},
revokeRolesFromRole: {skip: "primary only"},
revokeRolesFromUser: {skip: "primary only"},
rewriteCollection: {skip: "primary only"},
rolesInfo: {skip: "primary only"},
rotateCertificates: {skip: "does not return user data"},
rotateFTDC: {skip: "does not return user data"},

View File

@ -181,6 +181,7 @@ enums:
revokePrivilegesFromRole: "revokePrivilegesFromRole" # ID only
revokeRolesFromRole: "revokeRolesFromRole" # ID only
revokeRolesFromUser: "revokeRolesFromUser" # ID only
rewriteCollection: "rewriteCollection"
rotateCertificates: "rotateCertificates"
runAsLessPrivilegedUser: "runAsLessPrivilegedUser"
serverStatus: "serverStatus"

View File

@ -142,6 +142,7 @@ roles:
- moveCollection
- refineCollectionShardKey
- reshardCollection
- rewriteCollection
- unshardCollection
readAnyDatabase:
@ -384,6 +385,7 @@ roles:
- refineCollectionShardKey
- reshardCollection
- unshardCollection
- rewriteCollection
- matchType: any_system_buckets
actions: *clusterManagerRoleDatabaseActions

View File

@ -156,6 +156,7 @@ TEST(BuiltinRoles, addSystemBucketsPrivilegesForBuiltinRoleClusterManager) {
ActionType::splitVector,
ActionType::refineCollectionShardKey,
ActionType::reshardCollection,
ActionType::rewriteCollection,
ActionType::analyzeShardKey,
ActionType::configureQueryAnalyzer,
ActionType::unshardCollection,

View File

@ -32,3 +32,6 @@ filters:
- "*unshard*collection*":
approvers:
- 10gen/server-cluster-scalability
- "*rewrite*collection*":
approvers:
- 10gen/server-cluster-scalability

View File

@ -0,0 +1,116 @@
/**
* Copyright (C) 2025-present MongoDB, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
* linked combinations including the program with the OpenSSL library. You
* must comply with the Server Side Public License in all respects for
* all of the code used other than as permitted herein. If you modify file(s)
* with this exception, you may extend this exception to your version of the
* file(s), but you are not obligated to do so. If you do not wish to do so,
* delete this exception statement from your version. If you delete this
* exception statement from all source files in the program, then also delete
* it in the license file.
*/
#include "mongo/base/error_codes.h"
#include "mongo/db/auth/action_type.h"
#include "mongo/db/auth/authorization_session.h"
#include "mongo/db/auth/resource_pattern.h"
#include "mongo/db/commands.h"
#include "mongo/db/generic_argument_util.h"
#include "mongo/db/namespace_string.h"
#include "mongo/db/operation_context.h"
#include "mongo/db/service_context.h"
#include "mongo/db/sharding_environment/client/shard.h"
#include "mongo/db/sharding_environment/grid.h"
#include "mongo/db/topology/shard_registry.h"
#include "mongo/logv2/log.h"
#include "mongo/s/request_types/abort_reshard_collection_gen.h"
#include "mongo/util/assert_util.h"
#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand
namespace mongo {
namespace {
class ClusterAbortRewriteCollectionCmd : public TypedCommand<ClusterAbortRewriteCollectionCmd> {
public:
using Request = AbortRewriteCollection;
class Invocation final : public InvocationBase {
public:
using InvocationBase::InvocationBase;
void typedRun(OperationContext* opCtx) {
const NamespaceString& nss = ns();
LOGV2(8328901, "Beginning rewrite collection abort operation", logAttrs(ns()));
ConfigsvrAbortReshardCollection configsvrAbortReshardCollection(nss);
configsvrAbortReshardCollection.setDbName(request().getDbName());
configsvrAbortReshardCollection.setProvenance(
ReshardingProvenanceEnum::kRewriteCollection);
generic_argument_util::setMajorityWriteConcern(configsvrAbortReshardCollection,
&opCtx->getWriteConcern());
auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
auto cmdResponse = uassertStatusOK(
configShard->runCommand(opCtx,
ReadPreferenceSetting(ReadPreference::PrimaryOnly),
DatabaseName::kAdmin,
configsvrAbortReshardCollection.toBSON(),
Shard::RetryPolicy::kIdempotent));
uassertStatusOK(cmdResponse.commandStatus);
uassertStatusOK(cmdResponse.writeConcernStatus);
}
private:
NamespaceString ns() const override {
return request().getCommandParameter();
}
bool supportsWriteConcern() const override {
return false;
}
void doCheckAuthorization(OperationContext* opCtx) const override {
uassert(ErrorCodes::Unauthorized,
"Unauthorized",
AuthorizationSession::get(opCtx->getClient())
->isAuthorizedForActionsOnResource(ResourcePattern::forExactNamespace(ns()),
ActionType::rewriteCollection));
}
};
AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
return AllowedOnSecondary::kNever;
}
bool adminOnly() const override {
return true;
}
std::string help() const override {
return "Abort an in-progress rewrite collection operation for this collection.";
}
};
MONGO_REGISTER_COMMAND(ClusterAbortRewriteCollectionCmd).forRouter();
} // namespace
} // namespace mongo

View File

@ -0,0 +1,147 @@
/**
* Copyright (C) 2025-present MongoDB, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
* linked combinations including the program with the OpenSSL library. You
* must comply with the Server Side Public License in all respects for
* all of the code used other than as permitted herein. If you modify file(s)
* with this exception, you may extend this exception to your version of the
* file(s), but you are not obligated to do so. If you do not wish to do so,
* delete this exception statement from your version. If you delete this
* exception statement from all source files in the program, then also delete
* it in the license file.
*/
#include "mongo/db/auth/authorization_session.h"
#include "mongo/db/commands.h"
#include "mongo/db/generic_argument_util.h"
#include "mongo/db/global_catalog/ddl/sharded_ddl_commands_gen.h"
#include "mongo/db/global_catalog/router_role_api/cluster_commands_helpers.h"
#include "mongo/db/sharding_environment/grid.h"
#include "mongo/logv2/log.h"
#include "mongo/s/request_types/reshard_collection_gen.h"
#include "mongo/s/resharding/resharding_feature_flag_gen.h"
#include "mongo/util/assert_util.h"
#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand
namespace mongo {
namespace {
class ClusterRewriteCollectionCmd final : public TypedCommand<ClusterRewriteCollectionCmd> {
public:
using Request = RewriteCollection;
class Invocation final : public InvocationBase {
public:
using InvocationBase::InvocationBase;
void typedRun(OperationContext* opCtx) {
const auto& nss = ns();
ReshardCollectionRequest reshardCollectionRequest;
// ForceRedistribution is set to true to force resharding.
reshardCollectionRequest.setForceRedistribution(true);
// Retrieve the current shard key and pass it to reshardCollection. If shard key becomes
// stale due to a concurrent operation, the ReshardCollectionCoordinator will detect it
// and throw an error.
auto catalogClient = Grid::get(opCtx)->catalogClient();
CollectionType coll = catalogClient->getCollection(opCtx, nss);
reshardCollectionRequest.setKey(coll.getKeyPattern().toBSON());
// Other parameter values are passed through to reshardCollection.
reshardCollectionRequest.setZones(request().getZones());
reshardCollectionRequest.setNumInitialChunks(request().getNumInitialChunks());
reshardCollectionRequest.setPerformVerification(request().getPerformVerification());
if (resharding::gfeatureFlagReshardingNumSamplesPerChunk.isEnabled(
VersionContext::getDecoration(opCtx),
serverGlobalParams.featureCompatibility.acquireFCVSnapshot())) {
reshardCollectionRequest.setNumSamplesPerChunk(request().getNumSamplesPerChunk());
}
reshardCollectionRequest.setDemoMode(request().getDemoMode());
reshardCollectionRequest.setProvenance(ReshardingProvenanceEnum::kRewriteCollection);
ShardsvrReshardCollection rewriteCollectionRequest(nss);
rewriteCollectionRequest.setDbName(request().getDbName());
rewriteCollectionRequest.setReshardCollectionRequest(
std::move(reshardCollectionRequest));
generic_argument_util::setMajorityWriteConcern(rewriteCollectionRequest,
&opCtx->getWriteConcern());
LOGV2(8328900,
"Running a reshard collection command for the rewrite collection request.",
"dbName"_attr = request().getDbName());
sharding::router::DBPrimaryRouter router(opCtx->getServiceContext(), nss.dbName());
router.route(opCtx,
Request::kCommandParameterFieldName,
[&](OperationContext* opCtx, const CachedDatabaseInfo& dbInfo) {
auto cmdResponse =
executeCommandAgainstDatabasePrimaryOnlyAttachingDbVersion(
opCtx,
DatabaseName::kAdmin,
dbInfo,
rewriteCollectionRequest.toBSON(),
ReadPreferenceSetting(ReadPreference::PrimaryOnly),
Shard::RetryPolicy::kIdempotent);
const auto remoteResponse = uassertStatusOK(cmdResponse.swResponse);
uassertStatusOK(getStatusFromCommandResult(remoteResponse.data));
});
}
private:
NamespaceString ns() const override {
return request().getCommandParameter();
}
bool supportsWriteConcern() const override {
return false;
}
void doCheckAuthorization(OperationContext* opCtx) const override {
uassert(ErrorCodes::Unauthorized,
"Unauthorized",
AuthorizationSession::get(opCtx->getClient())
->isAuthorizedForActionsOnResource(ResourcePattern::forExactNamespace(ns()),
ActionType::rewriteCollection));
}
};
AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
return AllowedOnSecondary::kNever;
}
bool adminOnly() const override {
return true;
}
std::string help() const override {
return "Rewrite a sharded collection on its existing shard key.";
}
};
MONGO_REGISTER_COMMAND(ClusterRewriteCollectionCmd).forRouter();
} // namespace
} // namespace mongo

View File

@ -89,11 +89,13 @@ const auto kReportedStateFieldNamesMap = [] {
struct Metrics {
ReshardingCumulativeMetrics _resharding;
ReshardingCumulativeMetrics _moveCollection;
ReshardingCumulativeMetrics _rewriteCollection;
ReshardingCumulativeMetrics _balancerMoveCollection;
ReshardingCumulativeMetrics _unshardCollection;
Metrics()
: _moveCollection{"moveCollection"},
_rewriteCollection("rewriteCollection"),
_balancerMoveCollection{"balancerMoveCollection"},
_unshardCollection{"unshardCollection"} {};
};
@ -119,6 +121,12 @@ ReshardingCumulativeMetrics* ReshardingCumulativeMetrics::getForMoveCollection(
return &metrics->_moveCollection;
}
ReshardingCumulativeMetrics* ReshardingCumulativeMetrics::getForRewriteCollection(
ServiceContext* context) {
auto& metrics = getMetrics(context);
return &metrics->_rewriteCollection;
}
ReshardingCumulativeMetrics* ReshardingCumulativeMetrics::getForBalancerMoveCollection(
ServiceContext* context) {
auto& metrics = getMetrics(context);

View File

@ -88,6 +88,7 @@ public:
static ReshardingCumulativeMetrics* getForResharding(ServiceContext* context);
static ReshardingCumulativeMetrics* getForMoveCollection(ServiceContext* context);
static ReshardingCumulativeMetrics* getForRewriteCollection(ServiceContext* context);
static ReshardingCumulativeMetrics* getForBalancerMoveCollection(ServiceContext* context);
static ReshardingCumulativeMetrics* getForUnshardCollection(ServiceContext* context);

View File

@ -180,6 +180,8 @@ public:
switch (provenance) {
case ReshardingProvenanceEnum::kMoveCollection:
return ReshardingCumulativeMetrics::getForMoveCollection(serviceContext);
case ReshardingProvenanceEnum::kRewriteCollection:
return ReshardingCumulativeMetrics::getForRewriteCollection(serviceContext);
case ReshardingProvenanceEnum::kBalancerMoveCollection:
return ReshardingCumulativeMetrics::getForBalancerMoveCollection(
serviceContext);

View File

@ -183,6 +183,7 @@ public:
using Metrics = ReshardingCumulativeMetrics;
Metrics::getForResharding(sCtx)->reportForServerStatus(bob);
Metrics::getForMoveCollection(sCtx)->reportForServerStatus(bob);
Metrics::getForRewriteCollection(sCtx)->reportForServerStatus(bob);
Metrics::getForBalancerMoveCollection(sCtx)->reportForServerStatus(bob);
Metrics::getForUnshardCollection(sCtx)->reportForServerStatus(bob);
}

View File

@ -47,6 +47,7 @@ enums:
kMoveCollection: "moveCollection"
kUnshardCollection: "unshardCollection"
kBalancerMoveCollection: "balancerMoveCollection"
kRewriteCollection: "rewriteCollection"
types:
shard_id:

View File

@ -121,6 +121,7 @@ mongo_cc_library(
"//src/mongo/db/cluster_parameters:cluster_get_cluster_parameter_cmd.cpp",
"//src/mongo/db/cluster_parameters:cluster_set_cluster_parameter_cmd.cpp",
"//src/mongo/db/global_catalog/ddl:cluster_abort_move_collection_cmd.cpp",
"//src/mongo/db/global_catalog/ddl:cluster_abort_rewrite_collection_cmd.cpp",
"//src/mongo/db/global_catalog/ddl:cluster_abort_unshard_collection_cmd.cpp",
"//src/mongo/db/global_catalog/ddl:cluster_collection_mod_cmd.cpp",
"//src/mongo/db/global_catalog/ddl:cluster_commit_reshard_collection_cmd.cpp",
@ -139,6 +140,7 @@ mongo_cc_library(
"//src/mongo/db/global_catalog/ddl:cluster_rename_collection_cmd.cpp",
"//src/mongo/db/global_catalog/ddl:cluster_repair_sharded_collection_chunks_history_cmd.cpp",
"//src/mongo/db/global_catalog/ddl:cluster_reset_placement_history_cmd.cpp",
"//src/mongo/db/global_catalog/ddl:cluster_rewrite_collection_cmd.cpp",
"//src/mongo/db/global_catalog/ddl:cluster_set_allow_migrations_cmd.cpp",
"//src/mongo/db/global_catalog/ddl:cluster_shard_collection_cmd.cpp",
"//src/mongo/db/global_catalog/ddl:cluster_unshard_collection_cmd.cpp",

View File

@ -81,6 +81,14 @@ commands:
api_version: ""
type: namespacestring
abortRewriteCollection:
description: "The public command on mongos that aborts a rewriteCollection commmand."
command_name: abortRewriteCollection
strict: false
namespace: type
api_version: ""
type: namespacestring
abortUnshardCollection:
description: "The public command on mongos that aborts a unshardCollection commmand."
command_name: abortUnshardCollection

View File

@ -197,6 +197,37 @@ commands:
and reshardingDelayBeforeRemainingOperationTimeQueryMillis values to 0
for quick demo of reshardCollection operation"
rewriteCollection:
description: "The public command on mongos that rewrites a sharded collection."
command_name: rewriteCollection
strict: true
namespace: type
api_version: ""
type: namespacestring
fields:
numInitialChunks:
type: safeInt64
description: "The number of chunks to create initially."
optional: true
zones:
type: array<ReshardingZoneType>
description: "The zones for the new shard key."
optional: true
performVerification:
type: bool
description: "Whether to perform data comparison verification."
optional: true
numSamplesPerChunk:
type: safeInt64
description: "The number of documents to sample on new chunks"
optional: true
demoMode:
type: optionalBool
description: >-
"When set to true, overrides reshardingMinimumOperationDurationMillis
and reshardingDelayBeforeRemainingOperationTimeQueryMillis values to 0
for quick demo of reshardCollection operation"
moveCollection:
description: "The public command on mongos that moves one unsharded collection from source to destination shard."
command_name: moveCollection