mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-50792 Implement detailed shard key index errors

This commit is contained in:
Israel Hsu 2022-06-13 17:15:12 +00:00 committed by Evergreen Agent
parent 35eb6deef0
commit a98ea3b6e7
7 changed files with 243 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
jstests/core/check_shard_index.js
View File

@ -150,4 +150,69 @@ res = db.runCommand(
{checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1, z: 1}});
assert.eq(false, res.ok, "4e " + tojson(res));
// -------------------------
// Test error messages of checkShardingIndex failing:
// Shard key is not a prefix of index key:
f.drop();
f.createIndex({x: 1});
res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {y: 1}});
assert.eq(false, res.ok);
assert(res.errmsg.includes("Shard key is not a prefix of index key."));
// Index key is partial:
f.drop();
f.createIndex({x: 1, y: 1}, {partialFilterExpression: {y: {$gt: 0}}});
res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
assert.eq(false, res.ok);
assert(res.errmsg.includes("Index key is partial."));
// Index key is sparse:
f.drop();
f.createIndex({x: 1, y: 1}, {sparse: true});
res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
assert.eq(false, res.ok);
assert(res.errmsg.includes("Index key is sparse."));
// Index key is multikey:
f.drop();
f.createIndex({x: 1, y: 1});
f.save({y: [1, 2, 3, 4, 5]});
res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
assert.eq(false, res.ok);
assert(res.errmsg.includes("Index key is multikey."));
// Index key has a non-simple collation:
f.drop();
f.createIndex({x: 1, y: 1}, {collation: {locale: "en"}});
res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
assert.eq(false, res.ok);
assert(res.errmsg.includes("Index has a non-simple collation."));
// Index key is sparse and index has non-simple collation:
f.drop();
f.createIndex({x: 1, y: 1}, {sparse: true, collation: {locale: "en"}});
res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
assert.eq(false, res.ok);
assert(res.errmsg.includes("Index key is sparse.") &&
res.errmsg.includes("Index has a non-simple collation."));
// Multiple incompatible indexes: Index key is multikey and is partial:
f.drop();
f.createIndex({x: 1, y: 1}, {name: "index_1_part", partialFilterExpression: {x: {$gt: 0}}});
f.createIndex({x: 1, y: 1}, {name: "index_2"});
f.save({y: [1, 2, 3, 4, 5]});
res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
assert.eq(false, res.ok);
assert(res.errmsg.includes("Index key is multikey.") &&
res.errmsg.includes("Index key is partial."));
// Multiple incompatible indexes: Index key is partial and sparse:
f.drop();
f.createIndex({x: 1, y: 1}, {name: "index_1_part", partialFilterExpression: {x: {$gt: 0}}});
f.createIndex({x: 1, y: 1}, {name: "index_2_sparse", sparse: true});
res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
assert.eq(false, res.ok);
assert(res.errmsg.includes("Index key is partial.") && res.errmsg.includes("Index key is sparse."));
print("PASSED");

73
jstests/sharding/refine_collection_shard_key_basic.js
View File

@ -249,6 +249,8 @@ function validateUnrelatedCollAfterRefine(oldCollArr, oldChunkArr, oldTagsArr) {
jsTestLog('********** SIMPLE TESTS **********');
var result;
// Should fail because arguments 'refineCollectionShardKey' and 'key' are invalid types.
assert.commandFailedWithCode(
mongos.adminCommand({refineCollectionShardKey: {_id: 1}, key: {_id: 1, aKey: 1}}),
@ -355,36 +357,46 @@ assert.commandFailedWithCode(
dropAndReshardColl({_id: 1});
assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}, {sparse: true}));
assert.commandFailedWithCode(
mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
ErrorCodes.InvalidOptions);
result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
assert(result.errmsg.includes("Index key is sparse."));
// Should fail because index has a non-simple collation.
dropAndReshardColl({aKey: 1});
assert.commandWorked(mongos.getCollection(kNsName).createIndex({aKey: 1, bKey: 1}, {
collation: {
locale: "en",
}
}));
result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1, bKey: 1}});
assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
assert(result.errmsg.includes("Index has a non-simple collation."));
// Should fail because only a partial index exists for new shard key {_id: 1, aKey: 1}.
dropAndReshardColl({_id: 1});
assert.commandWorked(mongos.getCollection(kNsName).createIndex(
{_id: 1, aKey: 1}, {partialFilterExpression: {aKey: {$gt: 0}}}));
assert.commandFailedWithCode(
mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
ErrorCodes.InvalidOptions);
result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
assert(result.errmsg.includes("Index key is partial."));
// Should fail because only a multikey index exists for new shard key {_id: 1, aKey: 1}.
dropAndReshardColl({_id: 1});
assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}));
assert.commandWorked(mongos.getCollection(kNsName).insert({aKey: [1, 2, 3, 4, 5]}));
assert.commandFailedWithCode(
mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
ErrorCodes.InvalidOptions);
result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
assert(result.errmsg.includes("Index key is multikey."));
// Should fail because current shard key {a: 1} is unique, new shard key is {a: 1, b: 1}, and an
// index only exists on {a: 1, b: 1, c: 1}.
dropAndReshardCollUnique({a: 1});
assert.commandWorked(mongos.getCollection(kNsName).createIndex({a: 1, b: 1, c: 1}));
assert.commandFailedWithCode(
mongos.adminCommand({refineCollectionShardKey: kNsName, key: {a: 1, b: 1}}),
ErrorCodes.InvalidOptions);
mongos.adminCommand({refineCollectionShardKey: kNsName, key: {a: 1, b: 1}});
assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
// Should work because current shard key {_id: 1} is not unique, new shard key is {_id: 1, aKey:
// 1}, and an index exists on {_id: 1, aKey: 1, bKey: 1}.
@ -459,6 +471,43 @@ assert.commandFailedWithCode(
mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1, bKey: 1}}),
ErrorCodes.InvalidOptions);
// Should fail because index key is sparse and index has non-simple collation.
dropAndReshardColl({_id: 1});
assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}, {
sparse: true,
collation: {
locale: "en",
}
}));
result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
assert(result.errmsg.includes("Index key is sparse.") &&
result.errmsg.includes("Index has a non-simple collation."));
// Should fail because index key is multikey and is partial.
dropAndReshardColl({_id: 1});
assert.commandWorked(mongos.getCollection(kNsName).createIndex(
{_id: 1, aKey: 1}, {name: "index_1_part", partialFilterExpression: {aKey: {$gt: 0}}}));
assert.commandWorked(
mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}, {name: "index_2"}));
assert.commandWorked(mongos.getCollection(kNsName).insert({aKey: [1, 2, 3, 4, 5]}));
result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
assert(result.errmsg.includes("Index key is multikey.") &&
result.errmsg.includes("Index key is partial."));
// Should fail because both indexes have keys that are incompatible: partial; sparse
dropAndReshardColl({_id: 1});
assert.commandWorked(mongos.getCollection(kNsName).createIndex(
{_id: 1, aKey: 1}, {name: "index_1_part", partialFilterExpression: {aKey: {$gt: 0}}}));
assert.commandWorked(mongos.getCollection(kNsName).createIndex(
{_id: 1, aKey: 1}, {name: "index_2_sparse", sparse: true}));
result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
assert(result.errmsg.includes("Index key is partial.") &&
result.errmsg.includes("Index key is sparse."));
// Should work because a 'useful' index exists for new shard key {_id: 1, aKey: 1}.
dropAndReshardColl({_id: 1});
assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}));

8
src/mongo/db/s/check_sharding_index_command.cpp
View File

@ -27,7 +27,6 @@
* it in the license file.
*/
#include "mongo/platform/basic.h"
#include "mongo/db/auth/action_type.h"
@ -40,7 +39,6 @@
#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding
namespace mongo {
namespace {
@ -96,13 +94,15 @@ public:
return false;
}
std::string tmpErrMsg = "couldn't find valid index for shard key";
auto shardKeyIdx = findShardKeyPrefixedIndex(opCtx,
*collection,
collection->getIndexCatalog(),
keyPattern,
/*requireSingleKey=*/true);
/*requireSingleKey=*/true,
&tmpErrMsg);
if (!shardKeyIdx) {
errmsg = "couldn't find valid index for shard key";
errmsg = tmpErrMsg;
return false;
}

73
src/mongo/db/s/shard_key_index_util.cpp
View File

@ -48,7 +48,8 @@ boost::optional<ShardKeyIndex> _findShardKeyPrefixedIndex(
const IndexCatalog* indexCatalog,
const boost::optional<std::string>& excludeName,
const BSONObj& shardKey,
bool requireSingleKey) {
bool requireSingleKey,
std::string* errMsg = nullptr) {
if (collection->isClustered() &&
clustered_util::matchesClusterKey(shardKey, collection->getClusteredInfo())) {
auto clusteredIndexSpec = collection->getClusteredInfo()->getIndexSpec();
@ -67,7 +68,8 @@ boost::optional<ShardKeyIndex> _findShardKeyPrefixedIndex(
continue;
}
if (isCompatibleWithShardKey(opCtx, collection, indexEntry, shardKey, requireSingleKey)) {
if (isCompatibleWithShardKey(
opCtx, collection, indexEntry, shardKey, requireSingleKey, errMsg)) {
if (!indexEntry->isMultikey(opCtx, collection)) {
return ShardKeyIndex(indexDescriptor);
}
@ -108,26 +110,72 @@ bool isCompatibleWithShardKey(OperationContext* opCtx,
const CollectionPtr& collection,
const IndexCatalogEntry* indexEntry,
const BSONObj& shardKey,
bool requireSingleKey) {
bool requireSingleKey,
std::string* errMsg) {
// Return a descriptive error for each index that shares a prefix with shardKey but
// cannot be used for sharding.
const int kErrorPartial = 0x01;
const int kErrorSparse = 0x02;
const int kErrorMultikey = 0x04;
const int kErrorCollation = 0x08;
const int kErrorNotPrefix = 0x10;
int reasons = 0;
auto desc = indexEntry->descriptor();
bool hasSimpleCollation = desc->collation().isEmpty();
if (desc->isPartial() || desc->isSparse()) {
return false;
if (desc->isPartial()) {
reasons |= kErrorPartial;
}
if (desc->isSparse()) {
reasons |= kErrorSparse;
}
if (!shardKey.isPrefixOf(desc->keyPattern(), SimpleBSONElementComparator::kInstance)) {
return false;
reasons |= kErrorNotPrefix;
}
if (!indexEntry->isMultikey(opCtx, collection) && hasSimpleCollation) {
return true;
if (reasons == 0) { // that is, not partial index, not sparse, and not prefix, then:
if (!indexEntry->isMultikey(opCtx, collection)) {
if (hasSimpleCollation) {
return true;
}
} else {
reasons |= kErrorMultikey;
}
if (!requireSingleKey && hasSimpleCollation) {
return true;
}
}
if (!requireSingleKey && hasSimpleCollation) {
return true;
if (!hasSimpleCollation) {
reasons |= kErrorCollation;
}
if (errMsg && reasons != 0) {
std::string errors = "Index " + indexEntry->descriptor()->indexName() +
" cannot be used for sharding because:";
if (reasons & kErrorPartial) {
errors += " Index key is partial.";
}
if (reasons & kErrorSparse) {
errors += " Index key is sparse.";
}
if (reasons & kErrorMultikey) {
errors += " Index key is multikey.";
}
if (reasons & kErrorCollation) {
errors += " Index has a non-simple collation.";
}
if (reasons & kErrorNotPrefix) {
errors += " Shard key is not a prefix of index key.";
}
if (!errMsg->empty()) {
*errMsg += "\n";
}
*errMsg += errors;
}
return false;
}
@ -145,9 +193,10 @@ boost::optional<ShardKeyIndex> findShardKeyPrefixedIndex(OperationContext* opCtx
const CollectionPtr& collection,
const IndexCatalog* indexCatalog,
const BSONObj& shardKey,
bool requireSingleKey) {
bool requireSingleKey,
std::string* errMsg) {
return _findShardKeyPrefixedIndex(
opCtx, collection, indexCatalog, boost::none, shardKey, requireSingleKey);
opCtx, collection, indexCatalog, boost::none, shardKey, requireSingleKey, errMsg);
}
} // namespace mongo

9
src/mongo/db/s/shard_key_index_util.h
View File

@ -67,12 +67,16 @@ private:
/**
* Returns true if the given index is compatible with the shard key pattern.
*
* If return value is false and errMsg is non-null, the reasons that the existing index is
* incompatible will be appended to errMsg.
*/
bool isCompatibleWithShardKey(OperationContext* opCtx,
const CollectionPtr& collection,
const IndexCatalogEntry* indexEntry,
const BSONObj& shardKey,
bool requireSingleKey);
bool requireSingleKey,
std::string* errMsg = nullptr);
/**
* Returns an index suitable for shard key range scans if it exists.
@ -89,7 +93,8 @@ boost::optional<ShardKeyIndex> findShardKeyPrefixedIndex(OperationContext* opCtx
const CollectionPtr& collection,
const IndexCatalog* indexCatalog,
const BSONObj& shardKey,
bool requireSingleKey);
bool requireSingleKey,
std::string* errMsg = nullptr);
/**
* Returns true if the given index name is the last remaining index that is compatible with the

49
src/mongo/db/s/shard_key_util.cpp
View File

@ -107,7 +107,8 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
const ShardKeyPattern& shardKeyPattern,
const boost::optional<BSONObj>& defaultCollation,
bool requiresUnique,
const ShardKeyValidationBehaviors& behaviors) {
const ShardKeyValidationBehaviors& behaviors,
std::string* errMsg) {
auto indexes = behaviors.loadIndexes(nss);
// 1. Verify consistency with existing unique indexes
@ -124,7 +125,9 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
// 2. Check for a useful index
bool hasUsefulIndexForKey = false;
std::string allReasons;
for (const auto& idx : indexes) {
std::string reasons;
BSONObj currentKey = idx["key"].embeddedObject();
// Check 2.i. and 2.ii.
if (!idx["sparse"].trueValue() && idx["filter"].eoo() && idx["collation"].eoo() &&
@ -143,6 +146,19 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
idx["seed"].numberInt() == BSONElementHasher::DEFAULT_HASH_SEED);
hasUsefulIndexForKey = true;
}
if (idx["sparse"].trueValue()) {
reasons += " Index key is sparse.";
}
if (idx["filter"].ok()) {
reasons += " Index key is partial.";
}
if (idx["collation"].ok()) {
reasons += " Index has a non-simple collation.";
}
if (!reasons.empty()) {
allReasons =
" Index " + idx["name"] + " cannot be used for sharding because [" + reasons + " ]";
}
}
// 3. If proposed key is required to be unique, additionally check for exact match.
@ -173,6 +189,10 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
}
}
if (errMsg && !allReasons.empty()) {
*errMsg += allReasons;
}
if (hasUsefulIndexForKey) {
// Check 2.iii Make sure that there is a useful, non-multikey index available.
behaviors.verifyUsefulNonMultiKeyIndex(nss, shardKeyPattern.toBSON());
@ -188,17 +208,19 @@ bool validateShardKeyIndexExistsOrCreateIfPossible(OperationContext* opCtx,
bool unique,
bool enforceUniquenessCheck,
const ShardKeyValidationBehaviors& behaviors) {
std::string errMsg;
if (validShardKeyIndexExists(opCtx,
nss,
shardKeyPattern,
defaultCollation,
unique && enforceUniquenessCheck,
behaviors)) {
behaviors,
&errMsg)) {
return false;
}
// 4. If no useful index, verify we can create one.
behaviors.verifyCanCreateShardKeyIndex(nss);
behaviors.verifyCanCreateShardKeyIndex(nss, &errMsg);
// 5. If no useful index exists and we can create one, create one on proposedKey. Only need
// to call ensureIndex on primary shard, since indexes get copied to receiving shard
@ -271,11 +293,12 @@ void ValidationBehaviorsShardCollection::verifyUsefulNonMultiKeyIndex(
uassert(ErrorCodes::InvalidOptions, res["errmsg"].str(), success);
}
void ValidationBehaviorsShardCollection::verifyCanCreateShardKeyIndex(
const NamespaceString& nss) const {
void ValidationBehaviorsShardCollection::verifyCanCreateShardKeyIndex(const NamespaceString& nss,
std::string* errMsg) const {
uassert(ErrorCodes::InvalidOptions,
"Please create an index that starts with the proposed shard key before "
"sharding the collection",
str::stream() << "Please create an index that starts with the proposed shard key before"
" sharding the collection. "
<< *errMsg,
_localClient->findOne(nss, BSONObj{}).isEmpty());
}
@ -334,11 +357,13 @@ void ValidationBehaviorsRefineShardKey::verifyUsefulNonMultiKeyIndex(
uassertStatusOK(checkShardingIndexRes.commandStatus);
}
void ValidationBehaviorsRefineShardKey::verifyCanCreateShardKeyIndex(
const NamespaceString& nss) const {
uasserted(ErrorCodes::InvalidOptions,
"Please create an index that starts with the proposed shard key before "
"refining the shard key of the collection");
void ValidationBehaviorsRefineShardKey::verifyCanCreateShardKeyIndex(const NamespaceString& nss,
std::string* errMsg) const {
uasserted(
ErrorCodes::InvalidOptions,
str::stream() << "Please create an index that starts with the proposed shard key before"
" sharding the collection. "
<< *errMsg);
}
void ValidationBehaviorsRefineShardKey::createShardKeyIndex(

12
src/mongo/db/s/shard_key_util.h
View File

@ -51,7 +51,8 @@ public:
virtual void verifyUsefulNonMultiKeyIndex(const NamespaceString& nss,
const BSONObj& proposedKey) const = 0;
virtual void verifyCanCreateShardKeyIndex(const NamespaceString& nss) const = 0;
virtual void verifyCanCreateShardKeyIndex(const NamespaceString& nss,
std::string* errMsg) const = 0;
virtual void createShardKeyIndex(const NamespaceString& nss,
const BSONObj& proposedKey,
@ -72,7 +73,8 @@ public:
void verifyUsefulNonMultiKeyIndex(const NamespaceString& nss,
const BSONObj& proposedKey) const override;
void verifyCanCreateShardKeyIndex(const NamespaceString& nss) const override;
void verifyCanCreateShardKeyIndex(const NamespaceString& nss,
std::string* errMsg) const override;
void createShardKeyIndex(const NamespaceString& nss,
const BSONObj& proposedKey,
@ -95,7 +97,8 @@ public:
void verifyUsefulNonMultiKeyIndex(const NamespaceString& nss,
const BSONObj& proposedKey) const override;
void verifyCanCreateShardKeyIndex(const NamespaceString& nss) const override;
void verifyCanCreateShardKeyIndex(const NamespaceString& nss,
std::string* errMsg) const override;
void createShardKeyIndex(const NamespaceString& nss,
const BSONObj& proposedKey,
@ -165,7 +168,8 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
const ShardKeyPattern& shardKeyPattern,
const boost::optional<BSONObj>& defaultCollation,
bool requiresUnique,
const ShardKeyValidationBehaviors& behaviors);
const ShardKeyValidationBehaviors& behaviors,
std::string* errMsg = nullptr);
void validateShardKeyIsNotEncrypted(OperationContext* opCtx,
const NamespaceString& nss,