SERVER-50792 Implement detailed shard key index errors

jstests/core/check_shard_index.js

@@ -150,4 +150,69 @@ res = db.runCommand(
     {checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1, z: 1}});
 assert.eq(false, res.ok, "4e " + tojson(res));
 
+// -------------------------
+// Test error messages of checkShardingIndex failing:
+
+// Shard key is not a prefix of index key:
+f.drop();
+f.createIndex({x: 1});
+res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {y: 1}});
+assert.eq(false, res.ok);
+assert(res.errmsg.includes("Shard key is not a prefix of index key."));
+
+// Index key is partial:
+f.drop();
+f.createIndex({x: 1, y: 1}, {partialFilterExpression: {y: {$gt: 0}}});
+res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
+assert.eq(false, res.ok);
+assert(res.errmsg.includes("Index key is partial."));
+
+// Index key is sparse:
+f.drop();
+f.createIndex({x: 1, y: 1}, {sparse: true});
+res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
+assert.eq(false, res.ok);
+assert(res.errmsg.includes("Index key is sparse."));
+
+// Index key is multikey:
+f.drop();
+f.createIndex({x: 1, y: 1});
+f.save({y: [1, 2, 3, 4, 5]});
+res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
+assert.eq(false, res.ok);
+assert(res.errmsg.includes("Index key is multikey."));
+
+// Index key has a non-simple collation:
+f.drop();
+f.createIndex({x: 1, y: 1}, {collation: {locale: "en"}});
+res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
+assert.eq(false, res.ok);
+assert(res.errmsg.includes("Index has a non-simple collation."));
+
+// Index key is sparse and index has non-simple collation:
+f.drop();
+f.createIndex({x: 1, y: 1}, {sparse: true, collation: {locale: "en"}});
+res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
+assert.eq(false, res.ok);
+assert(res.errmsg.includes("Index key is sparse.") &&
+       res.errmsg.includes("Index has a non-simple collation."));
+
+// Multiple incompatible indexes: Index key is multikey and is partial:
+f.drop();
+f.createIndex({x: 1, y: 1}, {name: "index_1_part", partialFilterExpression: {x: {$gt: 0}}});
+f.createIndex({x: 1, y: 1}, {name: "index_2"});
+f.save({y: [1, 2, 3, 4, 5]});
+res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
+assert.eq(false, res.ok);
+assert(res.errmsg.includes("Index key is multikey.") &&
+       res.errmsg.includes("Index key is partial."));
+
+// Multiple incompatible indexes: Index key is partial and sparse:
+f.drop();
+f.createIndex({x: 1, y: 1}, {name: "index_1_part", partialFilterExpression: {x: {$gt: 0}}});
+f.createIndex({x: 1, y: 1}, {name: "index_2_sparse", sparse: true});
+res = db.runCommand({checkShardingIndex: "test.jstests_shardingindex", keyPattern: {x: 1, y: 1}});
+assert.eq(false, res.ok);
+assert(res.errmsg.includes("Index key is partial.") && res.errmsg.includes("Index key is sparse."));
+
 print("PASSED");

jstests/sharding/refine_collection_shard_key_basic.js

@@ -249,6 +249,8 @@ function validateUnrelatedCollAfterRefine(oldCollArr, oldChunkArr, oldTagsArr) {
 
 jsTestLog('********** SIMPLE TESTS **********');
 
+var result;
+
 // Should fail because arguments 'refineCollectionShardKey' and 'key' are invalid types.
 assert.commandFailedWithCode(
     mongos.adminCommand({refineCollectionShardKey: {_id: 1}, key: {_id: 1, aKey: 1}}),
@@ -355,36 +357,46 @@ assert.commandFailedWithCode(
 dropAndReshardColl({_id: 1});
 assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}, {sparse: true}));
 
-assert.commandFailedWithCode(
+result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
-    mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
+assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
-    ErrorCodes.InvalidOptions);
+assert(result.errmsg.includes("Index key is sparse."));
+
+// Should fail because index has a non-simple collation.
+dropAndReshardColl({aKey: 1});
+assert.commandWorked(mongos.getCollection(kNsName).createIndex({aKey: 1, bKey: 1}, {
+    collation: {
+        locale: "en",
+    }
+}));
+result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1, bKey: 1}});
+assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
+assert(result.errmsg.includes("Index has a non-simple collation."));
 
 // Should fail because only a partial index exists for new shard key {_id: 1, aKey: 1}.
 dropAndReshardColl({_id: 1});
 assert.commandWorked(mongos.getCollection(kNsName).createIndex(
     {_id: 1, aKey: 1}, {partialFilterExpression: {aKey: {$gt: 0}}}));
 
-assert.commandFailedWithCode(
+result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
-    mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
+assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
-    ErrorCodes.InvalidOptions);
+assert(result.errmsg.includes("Index key is partial."));
 
 // Should fail because only a multikey index exists for new shard key {_id: 1, aKey: 1}.
 dropAndReshardColl({_id: 1});
 assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}));
 assert.commandWorked(mongos.getCollection(kNsName).insert({aKey: [1, 2, 3, 4, 5]}));
 
-assert.commandFailedWithCode(
+result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
-    mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
+assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
-    ErrorCodes.InvalidOptions);
+assert(result.errmsg.includes("Index key is multikey."));
 
 // Should fail because current shard key {a: 1} is unique, new shard key is {a: 1, b: 1}, and an
 // index only exists on {a: 1, b: 1, c: 1}.
 dropAndReshardCollUnique({a: 1});
 assert.commandWorked(mongos.getCollection(kNsName).createIndex({a: 1, b: 1, c: 1}));
 
-assert.commandFailedWithCode(
+mongos.adminCommand({refineCollectionShardKey: kNsName, key: {a: 1, b: 1}});
-    mongos.adminCommand({refineCollectionShardKey: kNsName, key: {a: 1, b: 1}}),
+assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
-    ErrorCodes.InvalidOptions);
 
 // Should work because current shard key {_id: 1} is not unique, new shard key is {_id: 1, aKey:
 // 1}, and an index exists on {_id: 1, aKey: 1, bKey: 1}.
@@ -459,6 +471,43 @@ assert.commandFailedWithCode(
     mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1, bKey: 1}}),
     ErrorCodes.InvalidOptions);
 
+// Should fail because index key is sparse and index has non-simple collation.
+dropAndReshardColl({_id: 1});
+assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}, {
+    sparse: true,
+    collation: {
+        locale: "en",
+    }
+}));
+result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
+assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
+assert(result.errmsg.includes("Index key is sparse.") &&
+       result.errmsg.includes("Index has a non-simple collation."));
+
+// Should fail because index key is multikey and is partial.
+dropAndReshardColl({_id: 1});
+assert.commandWorked(mongos.getCollection(kNsName).createIndex(
+    {_id: 1, aKey: 1}, {name: "index_1_part", partialFilterExpression: {aKey: {$gt: 0}}}));
+assert.commandWorked(
+    mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}, {name: "index_2"}));
+assert.commandWorked(mongos.getCollection(kNsName).insert({aKey: [1, 2, 3, 4, 5]}));
+
+result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
+assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
+assert(result.errmsg.includes("Index key is multikey.") &&
+       result.errmsg.includes("Index key is partial."));
+
+// Should fail because both indexes have keys that are incompatible: partial; sparse
+dropAndReshardColl({_id: 1});
+assert.commandWorked(mongos.getCollection(kNsName).createIndex(
+    {_id: 1, aKey: 1}, {name: "index_1_part", partialFilterExpression: {aKey: {$gt: 0}}}));
+assert.commandWorked(mongos.getCollection(kNsName).createIndex(
+    {_id: 1, aKey: 1}, {name: "index_2_sparse", sparse: true}));
+result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}});
+assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions);
+assert(result.errmsg.includes("Index key is partial.") &&
+       result.errmsg.includes("Index key is sparse."));
+
 // Should work because a 'useful' index exists for new shard key {_id: 1, aKey: 1}.
 dropAndReshardColl({_id: 1});
 assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}));

src/mongo/db/s/check_sharding_index_command.cpp

@@ -27,7 +27,6 @@
  *    it in the license file.
  */
 
-
 #include "mongo/platform/basic.h"
 
 #include "mongo/db/auth/action_type.h"
@@ -40,7 +39,6 @@
 
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding
 
-
 namespace mongo {
 namespace {
 
@@ -96,13 +94,15 @@ public:
             return false;
         }
 
+        std::string tmpErrMsg = "couldn't find valid index for shard key";
         auto shardKeyIdx = findShardKeyPrefixedIndex(opCtx,
                                                      *collection,
                                                      collection->getIndexCatalog(),
                                                      keyPattern,
-                                                     /*requireSingleKey=*/true);
+                                                     /*requireSingleKey=*/true,
+                                                     &tmpErrMsg);
         if (!shardKeyIdx) {
-            errmsg = "couldn't find valid index for shard key";
+            errmsg = tmpErrMsg;
             return false;
         }
 

src/mongo/db/s/shard_key_index_util.cpp

@@ -48,7 +48,8 @@ boost::optional<ShardKeyIndex> _findShardKeyPrefixedIndex(
     const IndexCatalog* indexCatalog,
     const boost::optional<std::string>& excludeName,
     const BSONObj& shardKey,
-    bool requireSingleKey) {
+    bool requireSingleKey,
+    std::string* errMsg = nullptr) {
     if (collection->isClustered() &&
         clustered_util::matchesClusterKey(shardKey, collection->getClusteredInfo())) {
         auto clusteredIndexSpec = collection->getClusteredInfo()->getIndexSpec();
@@ -67,7 +68,8 @@ boost::optional<ShardKeyIndex> _findShardKeyPrefixedIndex(
             continue;
         }
 
-        if (isCompatibleWithShardKey(opCtx, collection, indexEntry, shardKey, requireSingleKey)) {
+        if (isCompatibleWithShardKey(
+                opCtx, collection, indexEntry, shardKey, requireSingleKey, errMsg)) {
             if (!indexEntry->isMultikey(opCtx, collection)) {
                 return ShardKeyIndex(indexDescriptor);
             }
@@ -108,26 +110,72 @@ bool isCompatibleWithShardKey(OperationContext* opCtx,
                               const CollectionPtr& collection,
                               const IndexCatalogEntry* indexEntry,
                               const BSONObj& shardKey,
-                              bool requireSingleKey) {
+                              bool requireSingleKey,
+                              std::string* errMsg) {
+    // Return a descriptive error for each index that shares a prefix with shardKey but
+    // cannot be used for sharding.
+    const int kErrorPartial = 0x01;
+    const int kErrorSparse = 0x02;
+    const int kErrorMultikey = 0x04;
+    const int kErrorCollation = 0x08;
+    const int kErrorNotPrefix = 0x10;
+    int reasons = 0;
+
     auto desc = indexEntry->descriptor();
     bool hasSimpleCollation = desc->collation().isEmpty();
 
-    if (desc->isPartial() || desc->isSparse()) {
+    if (desc->isPartial()) {
-        return false;
+        reasons |= kErrorPartial;
+    }
+
+    if (desc->isSparse()) {
+        reasons |= kErrorSparse;
     }
 
     if (!shardKey.isPrefixOf(desc->keyPattern(), SimpleBSONElementComparator::kInstance)) {
-        return false;
+        reasons |= kErrorNotPrefix;
     }
 
-    if (!indexEntry->isMultikey(opCtx, collection) && hasSimpleCollation) {
+    if (reasons == 0) {  // that is, not partial index, not sparse, and not prefix, then:
-        return true;
+        if (!indexEntry->isMultikey(opCtx, collection)) {
+            if (hasSimpleCollation) {
+                return true;
+            }
+        } else {
+            reasons |= kErrorMultikey;
+        }
+        if (!requireSingleKey && hasSimpleCollation) {
+            return true;
+        }
     }
 
-    if (!requireSingleKey && hasSimpleCollation) {
+    if (!hasSimpleCollation) {
-        return true;
+        reasons |= kErrorCollation;
     }
 
+    if (errMsg && reasons != 0) {
+        std::string errors = "Index " + indexEntry->descriptor()->indexName() +
+            " cannot be used for sharding because:";
+        if (reasons & kErrorPartial) {
+            errors += " Index key is partial.";
+        }
+        if (reasons & kErrorSparse) {
+            errors += " Index key is sparse.";
+        }
+        if (reasons & kErrorMultikey) {
+            errors += " Index key is multikey.";
+        }
+        if (reasons & kErrorCollation) {
+            errors += " Index has a non-simple collation.";
+        }
+        if (reasons & kErrorNotPrefix) {
+            errors += " Shard key is not a prefix of index key.";
+        }
+        if (!errMsg->empty()) {
+            *errMsg += "\n";
+        }
+        *errMsg += errors;
+    }
     return false;
 }
 
@@ -145,9 +193,10 @@ boost::optional<ShardKeyIndex> findShardKeyPrefixedIndex(OperationContext* opCtx
                                                          const CollectionPtr& collection,
                                                          const IndexCatalog* indexCatalog,
                                                          const BSONObj& shardKey,
-                                                         bool requireSingleKey) {
+                                                         bool requireSingleKey,
+                                                         std::string* errMsg) {
     return _findShardKeyPrefixedIndex(
-        opCtx, collection, indexCatalog, boost::none, shardKey, requireSingleKey);
+        opCtx, collection, indexCatalog, boost::none, shardKey, requireSingleKey, errMsg);
 }
 
 }  // namespace mongo

src/mongo/db/s/shard_key_index_util.h

@@ -67,12 +67,16 @@ private:
 
 /**
  * Returns true if the given index is compatible with the shard key pattern.
+ *
+ * If return value is false and errMsg is non-null, the reasons that the existing index is
+ * incompatible will be appended to errMsg.
  */
 bool isCompatibleWithShardKey(OperationContext* opCtx,
                               const CollectionPtr& collection,
                               const IndexCatalogEntry* indexEntry,
                               const BSONObj& shardKey,
-                              bool requireSingleKey);
+                              bool requireSingleKey,
+                              std::string* errMsg = nullptr);
 
 /**
  * Returns an index suitable for shard key range scans if it exists.
@@ -89,7 +93,8 @@ boost::optional<ShardKeyIndex> findShardKeyPrefixedIndex(OperationContext* opCtx
                                                          const CollectionPtr& collection,
                                                          const IndexCatalog* indexCatalog,
                                                          const BSONObj& shardKey,
-                                                         bool requireSingleKey);
+                                                         bool requireSingleKey,
+                                                         std::string* errMsg = nullptr);
 
 /**
  * Returns true if the given index name is the last remaining index that is compatible with the

src/mongo/db/s/shard_key_util.cpp

@@ -107,7 +107,8 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
                               const ShardKeyPattern& shardKeyPattern,
                               const boost::optional<BSONObj>& defaultCollation,
                               bool requiresUnique,
-                              const ShardKeyValidationBehaviors& behaviors) {
+                              const ShardKeyValidationBehaviors& behaviors,
+                              std::string* errMsg) {
     auto indexes = behaviors.loadIndexes(nss);
 
     // 1.  Verify consistency with existing unique indexes
@@ -124,7 +125,9 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
 
     // 2. Check for a useful index
     bool hasUsefulIndexForKey = false;
+    std::string allReasons;
     for (const auto& idx : indexes) {
+        std::string reasons;
         BSONObj currentKey = idx["key"].embeddedObject();
         // Check 2.i. and 2.ii.
         if (!idx["sparse"].trueValue() && idx["filter"].eoo() && idx["collation"].eoo() &&
@@ -143,6 +146,19 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
                         idx["seed"].numberInt() == BSONElementHasher::DEFAULT_HASH_SEED);
             hasUsefulIndexForKey = true;
         }
+        if (idx["sparse"].trueValue()) {
+            reasons += " Index key is sparse.";
+        }
+        if (idx["filter"].ok()) {
+            reasons += " Index key is partial.";
+        }
+        if (idx["collation"].ok()) {
+            reasons += " Index has a non-simple collation.";
+        }
+        if (!reasons.empty()) {
+            allReasons =
+                " Index " + idx["name"] + " cannot be used for sharding because [" + reasons + " ]";
+        }
     }
 
     // 3. If proposed key is required to be unique, additionally check for exact match.
@@ -173,6 +189,10 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
         }
     }
 
+    if (errMsg && !allReasons.empty()) {
+        *errMsg += allReasons;
+    }
+
     if (hasUsefulIndexForKey) {
         // Check 2.iii Make sure that there is a useful, non-multikey index available.
         behaviors.verifyUsefulNonMultiKeyIndex(nss, shardKeyPattern.toBSON());
@@ -188,17 +208,19 @@ bool validateShardKeyIndexExistsOrCreateIfPossible(OperationContext* opCtx,
                                                    bool unique,
                                                    bool enforceUniquenessCheck,
                                                    const ShardKeyValidationBehaviors& behaviors) {
+    std::string errMsg;
     if (validShardKeyIndexExists(opCtx,
                                  nss,
                                  shardKeyPattern,
                                  defaultCollation,
                                  unique && enforceUniquenessCheck,
-                                 behaviors)) {
+                                 behaviors,
+                                 &errMsg)) {
         return false;
     }
 
     // 4. If no useful index, verify we can create one.
-    behaviors.verifyCanCreateShardKeyIndex(nss);
+    behaviors.verifyCanCreateShardKeyIndex(nss, &errMsg);
 
     // 5. If no useful index exists and we can create one, create one on proposedKey. Only need
     //    to call ensureIndex on primary shard, since indexes get copied to receiving shard
@@ -271,11 +293,12 @@ void ValidationBehaviorsShardCollection::verifyUsefulNonMultiKeyIndex(
     uassert(ErrorCodes::InvalidOptions, res["errmsg"].str(), success);
 }
 
-void ValidationBehaviorsShardCollection::verifyCanCreateShardKeyIndex(
+void ValidationBehaviorsShardCollection::verifyCanCreateShardKeyIndex(const NamespaceString& nss,
-    const NamespaceString& nss) const {
+                                                                      std::string* errMsg) const {
     uassert(ErrorCodes::InvalidOptions,
-            "Please create an index that starts with the proposed shard key before "
+            str::stream() << "Please create an index that starts with the proposed shard key before"
-            "sharding the collection",
+                             " sharding the collection. "
+                          << *errMsg,
             _localClient->findOne(nss, BSONObj{}).isEmpty());
 }
 
@@ -334,11 +357,13 @@ void ValidationBehaviorsRefineShardKey::verifyUsefulNonMultiKeyIndex(
     uassertStatusOK(checkShardingIndexRes.commandStatus);
 }
 
-void ValidationBehaviorsRefineShardKey::verifyCanCreateShardKeyIndex(
+void ValidationBehaviorsRefineShardKey::verifyCanCreateShardKeyIndex(const NamespaceString& nss,
-    const NamespaceString& nss) const {
+                                                                     std::string* errMsg) const {
-    uasserted(ErrorCodes::InvalidOptions,
+    uasserted(
-              "Please create an index that starts with the proposed shard key before "
+        ErrorCodes::InvalidOptions,
-              "refining the shard key of the collection");
+        str::stream() << "Please create an index that starts with the proposed shard key before"
+                         " sharding the collection. "
+                      << *errMsg);
 }
 
 void ValidationBehaviorsRefineShardKey::createShardKeyIndex(

src/mongo/db/s/shard_key_util.h

@@ -51,7 +51,8 @@ public:
     virtual void verifyUsefulNonMultiKeyIndex(const NamespaceString& nss,
                                               const BSONObj& proposedKey) const = 0;
 
-    virtual void verifyCanCreateShardKeyIndex(const NamespaceString& nss) const = 0;
+    virtual void verifyCanCreateShardKeyIndex(const NamespaceString& nss,
+                                              std::string* errMsg) const = 0;
 
     virtual void createShardKeyIndex(const NamespaceString& nss,
                                      const BSONObj& proposedKey,
@@ -72,7 +73,8 @@ public:
     void verifyUsefulNonMultiKeyIndex(const NamespaceString& nss,
                                       const BSONObj& proposedKey) const override;
 
-    void verifyCanCreateShardKeyIndex(const NamespaceString& nss) const override;
+    void verifyCanCreateShardKeyIndex(const NamespaceString& nss,
+                                      std::string* errMsg) const override;
 
     void createShardKeyIndex(const NamespaceString& nss,
                              const BSONObj& proposedKey,
@@ -95,7 +97,8 @@ public:
     void verifyUsefulNonMultiKeyIndex(const NamespaceString& nss,
                                       const BSONObj& proposedKey) const override;
 
-    void verifyCanCreateShardKeyIndex(const NamespaceString& nss) const override;
+    void verifyCanCreateShardKeyIndex(const NamespaceString& nss,
+                                      std::string* errMsg) const override;
 
     void createShardKeyIndex(const NamespaceString& nss,
                              const BSONObj& proposedKey,
@@ -165,7 +168,8 @@ bool validShardKeyIndexExists(OperationContext* opCtx,
                               const ShardKeyPattern& shardKeyPattern,
                               const boost::optional<BSONObj>& defaultCollation,
                               bool requiresUnique,
-                              const ShardKeyValidationBehaviors& behaviors);
+                              const ShardKeyValidationBehaviors& behaviors,
+                              std::string* errMsg = nullptr);
 
 void validateShardKeyIsNotEncrypted(OperationContext* opCtx,
                                     const NamespaceString& nss,