mirror
/
mongo
mirror of https://github.com/mongodb/mongo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SERVER-80177 Add option for validate to return schema validation violations as warnings 

GitOrigin-RevId: aa8ad8cd8a1080ba4545f0d8b1c6b810a6efbf43
This commit is contained in:
Binh Vo 2023-11-01 20:49:47 +00:00 committed by MongoDB Bot
parent 65e68ab558
commit deea0da483
14 changed files with 102 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
jstests/hooks/validate_collections.js
View File

@ -117,6 +117,7 @@ function CollectionValidator() {
// accurate.
enforceFastCount:
!TestData.skipEnforceFastCountOnValidate && !TestData.allowUncleanShutdowns,
warnOnSchemaValidation: true
});
if (validateRes.ok !== 1) {
return {ok: 0, host: host, validateRes: validateRes};

6
jstests/libs/override_methods/validate_collections_on_shutdown.js
View File

@ -125,7 +125,11 @@ MongoRunner.validateCollectionsCallback = function(port) {
for (let i = 0; i < dbNames.length; ++i) {
const dbName = dbNames[i];
cmds.then("validating " + dbName, function(conn) {
const validateOptions = {full: true, enforceFastCount: true};
const validateOptions = {
full: true,
enforceFastCount: true,
warnOnSchemaValidation: true
};
// TODO (SERVER-24266): Once fast counts are tolerant to unclean shutdowns, remove the
// check for TestData.allowUncleanShutdowns.
if (TestData.skipEnforceFastCountOnValidate || TestData.allowUncleanShutdowns) {

3
src/mongo/db/catalog/collection_validation.cpp
View File

@ -499,6 +499,7 @@ Status validate(OperationContext* opCtx,
const NamespaceString& nss,
ValidateMode mode,
RepairMode repairMode,
const AdditionalOptions& additionalOptions,
ValidateResults* results,
BSONObjBuilder* output,
bool logDiagnostics) {
@ -506,7 +507,7 @@ Status validate(OperationContext* opCtx,
// This is deliberately outside of the try-catch block, so that any errors thrown in the
// constructor fail the cmd, as opposed to returning OK with valid:false.
ValidateState validateState(opCtx, nss, mode, repairMode, logDiagnostics);
ValidateState validateState(opCtx, nss, mode, repairMode, additionalOptions, logDiagnostics);
const auto replCoord = repl::ReplicationCoordinator::get(opCtx);
// Check whether we are allowed to read from this node after acquiring our locks. If we are

8
src/mongo/db/catalog/collection_validation.h
View File

@ -93,6 +93,13 @@ enum class RepairMode {
kAdjustMultikey,
};
/**
* Additional validation options that can run in any mode.
*/
struct AdditionalOptions {
bool warnOnSchemaValidation = false; // only warn on schema validation failure
};
/**
* Expects the caller to hold no locks.
*
@ -104,6 +111,7 @@ Status validate(OperationContext* opCtx,
const NamespaceString& nss,
ValidateMode mode,
RepairMode repairMode,
const AdditionalOptions& additionalOptions,
ValidateResults* results,
BSONObjBuilder* output,
bool logDiagnostics);

2
src/mongo/db/catalog/collection_validation_test.cpp
View File

@ -95,6 +95,7 @@ std::vector<std::pair<BSONObj, ValidateResults>> foregroundValidate(
nss,
mode,
repairMode,
/*additionalOptions=*/{},
&validateResults,
&output,
/*logDiagnostics=*/false));
@ -167,6 +168,7 @@ std::vector<std::pair<BSONObj, ValidateResults>> backgroundValidate(const Namesp
nss,
CollectionValidation::ValidateMode::kBackground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&validateResults,
&output,
/*logDiagnostics=*/false));

2
src/mongo/db/catalog/column_index_consistency_test.cpp
View File

@ -57,11 +57,13 @@ protected:
const CollectionOptions defaultCollectionOptions;
ASSERT_OK(storageInterface()->createCollection(opCtx, kNss, defaultCollectionOptions));
CollectionValidation::AdditionalOptions additionalOptions;
_validateState = std::make_unique<CollectionValidation::ValidateState>(
opCtx,
kNss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kNone,
additionalOptions,
/*logDiagnostics=*/false);
};

2
src/mongo/db/catalog/validate_adaptor.cpp
View File

@ -134,7 +134,7 @@ void schemaValidationFailed(CollectionValidation::ValidateState* state,
// When testing is enabled, only warn about non-compliant documents to prevent test failures.
if (TestingProctor::instance().isEnabled() ||
Collection::SchemaValidationResult::kWarn == result) {
Collection::SchemaValidationResult::kWarn == result || state->warnOnSchemaValidation()) {
results->warnings.push_back(kSchemaValidationFailedReason);
} else if (Collection::SchemaValidationResult::kError == result) {
results->errors.push_back(kSchemaValidationFailedReason);

4
src/mongo/db/catalog/validate_state.cpp
View File

@ -59,6 +59,7 @@ ValidateState::ValidateState(OperationContext* opCtx,
const NamespaceString& nss,
ValidateMode mode,
RepairMode repairMode,
const AdditionalOptions& additionalOptions,
bool logDiagnostics)
: _nss(nss),
_mode(mode),
@ -118,6 +119,9 @@ ValidateState::ValidateState(OperationContext* opCtx,
}
}
// Return warnings instead of errors on schema validation failures.
_warnOnSchemaValidation = additionalOptions.warnOnSchemaValidation;
// RepairMode is incompatible with the ValidateModes kBackground and
// kForegroundFullEnforceFastCount.
if (fixErrors()) {

6
src/mongo/db/catalog/validate_state.h
View File

@ -56,6 +56,7 @@ public:
const NamespaceString& nss,
ValidateMode mode,
RepairMode repairMode,
const AdditionalOptions& additionalOptions,
bool logDiagnostics);
const NamespaceString& nss() const {
@ -191,6 +192,10 @@ public:
return _logDiagnostics;
}
bool warnOnSchemaValidation() const {
return _warnOnSchemaValidation;
}
boost::optional<Timestamp> getValidateTimestamp() {
return _validateTs;
}
@ -236,6 +241,7 @@ private:
bool _collectionSchemaViolated = false;
bool _timeseriesDataInconsistency = false;
bool _BSONDataNonConformant = false;
bool _warnOnSchemaValidation = false;
boost::optional<ShouldNotConflictWithSecondaryBatchApplicationBlock> _noPBWM;
boost::optional<Lock::GlobalLock> _globalLock;

8
src/mongo/db/catalog/validate_state_test.cpp
View File

@ -169,6 +169,7 @@ TEST_F(ValidateStateTest, NonExistentCollectionShouldThrowNamespaceNotFoundError
kNss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
/*logDiagnostics=*/false),
AssertionException,
ErrorCodes::NamespaceNotFound);
@ -178,6 +179,7 @@ TEST_F(ValidateStateTest, NonExistentCollectionShouldThrowNamespaceNotFoundError
kNss,
CollectionValidation::ValidateMode::kBackground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
/*logDiagnostics=*/false),
AssertionException,
ErrorCodes::NamespaceNotFound);
@ -201,6 +203,7 @@ TEST_F(ValidateStateDiskTest, UncheckpointedCollectionShouldThrowCursorNotFoundE
kNss,
CollectionValidation::ValidateMode::kBackground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
/*logDiagnostics=*/false);
ASSERT_THROWS_CODE(
validateState.initializeCursors(opCtx), AssertionException, ErrorCodes::CursorNotFound);
@ -227,6 +230,7 @@ TEST_F(ValidateStateTest, OpenCursorsOnAllIndexes) {
kNss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
/*logDiagnostics=*/false);
validateState.initializeCursors(opCtx);
@ -246,6 +250,7 @@ TEST_F(ValidateStateTest, OpenCursorsOnAllIndexes) {
kNss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
/*logDiagnostics=*/false);
validateState.initializeCursors(opCtx);
ASSERT_EQ(validateState.getIndexIdents().size(), 5);
@ -274,6 +279,7 @@ TEST_F(ValidateStateDiskTest, OpenCursorsOnCheckpointedIndexes) {
kNss,
CollectionValidation::ValidateMode::kBackground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
/*logDiagnostics=*/false);
validateState.initializeCursors(opCtx);
@ -312,6 +318,7 @@ TEST_F(ValidateStateDiskTest, CursorsAreNotOpenedAgainstCheckpointedIndexesThatW
kNss,
CollectionValidation::ValidateMode::kBackground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
/*logDiagnostics=*/false);
validateState.initializeCursors(opCtx);
ASSERT_EQ(validateState.getIndexIdents().size(), 3);
@ -326,6 +333,7 @@ TEST_F(ValidateStateDiskTest, CursorsAreNotOpenedAgainstCheckpointedIndexesThatW
kNss,
CollectionValidation::ValidateMode::kBackground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
/*logDiagnostics=*/false);
validateState.initializeCursors(opCtx);
ASSERT_EQ(validateState.getIndexIdents().size(), 3);

13
src/mongo/db/commands/validate.cpp
View File

@ -374,9 +374,18 @@ public:
PrepareConflictBehavior::kIgnoreConflictsAllowWrites);
}
CollectionValidation::AdditionalOptions additionalOptions;
additionalOptions.warnOnSchemaValidation = cmdObj["warnOnSchemaValidation"].trueValue();
ValidateResults validateResults;
Status status = CollectionValidation::validate(
opCtx, nss, mode, repairMode, &validateResults, &result, logDiagnostics);
Status status = CollectionValidation::validate(opCtx,
nss,
mode,
repairMode,
additionalOptions,
&validateResults,
&result,
logDiagnostics);
if (!status.isOK()) {
return CommandHelpers::appendCommandStatusNoThrow(result, status);
}

1
src/mongo/db/repair.cpp
View File

@ -239,6 +239,7 @@ Status repairCollection(OperationContext* opCtx,
nss,
CollectionValidation::ValidateMode::kForegroundFullIndexOnly,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&validateResults,
&output,
/*logDiagnostics=*/false);

1
src/mongo/db/repl/idempotency_test_fixture.cpp
View File

@ -408,6 +408,7 @@ CollectionState IdempotencyTest::validate(const NamespaceString& nss) {
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&validateResults,
&bob,
/*logDiagnostics=*/false));

52
src/mongo/dbtests/validate_tests.cpp
View File

@ -150,8 +150,14 @@ protected:
BSONObjBuilder output;
forceCheckpoint(_background);
ASSERT_OK(CollectionValidation::validate(
&_opCtx, _nss, mode, repairMode, &results, &output, kLogDiagnostics));
ASSERT_OK(CollectionValidation::validate(&_opCtx,
_nss,
mode,
repairMode,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
// Check if errors are reported if and only if valid is set to false.
ASSERT_EQ(results.valid, results.errors.empty());
@ -1267,6 +1273,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -1387,6 +1394,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -1478,6 +1486,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -1593,6 +1602,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -1622,6 +1632,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -1658,6 +1669,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -1777,6 +1789,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -1808,6 +1821,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -1840,6 +1854,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -1930,6 +1945,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -1961,6 +1977,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -1992,6 +2009,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -2145,6 +2163,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -2175,6 +2194,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -2228,6 +2248,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -2401,6 +2422,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -2434,6 +2456,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -2488,6 +2511,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -2755,6 +2779,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -2785,6 +2810,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -2839,6 +2865,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -2968,6 +2995,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -2999,6 +3027,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -3031,6 +3060,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -3448,6 +3478,7 @@ public:
_nss,
mode,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -3516,6 +3547,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -3547,6 +3579,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -3582,6 +3615,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -3613,6 +3647,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -3786,6 +3821,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -3815,6 +3851,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -3845,6 +3882,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -4004,6 +4042,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -4033,6 +4072,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -4062,6 +4102,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -4171,6 +4212,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kAdjustMultikey,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -4204,6 +4246,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForeground,
CollectionValidation::RepairMode::kAdjustMultikey,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -4277,6 +4320,7 @@ public:
_nss,
mode,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -4391,6 +4435,7 @@ public:
_nss,
mode,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -4502,6 +4547,7 @@ public:
_nss,
mode,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -4542,6 +4588,7 @@ public:
_nss,
mode,
CollectionValidation::RepairMode::kFixErrors,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));
@ -4669,6 +4716,7 @@ public:
_nss,
CollectionValidation::ValidateMode::kForegroundFull,
CollectionValidation::RepairMode::kNone,
/*additionalOptions=*/{},
&results,
&output,
kLogDiagnostics));