From 8df5d7bb8440bc449d69d084d554963fc79c23b7 Mon Sep 17 00:00:00 2001 From: Allison Easton Date: Tue, 16 Dec 2025 10:59:14 +0100 Subject: [PATCH] SERVER-113220 Improve observability for connections via the maintenance port (#44499) GitOrigin-RevId: 16af001a392fe8ff33b029922d442d0ddcc62feb --- .../network/maintenance_port_metrics.js | 86 +++++++++++++++++ .../noPassthrough/query/log_find_getmore.js | 63 ++++++++---- src/mongo/db/curop.cpp | 5 + src/mongo/db/curop_test.cpp | 95 +++++++++++++++++++ src/mongo/db/op_debug.cpp | 12 +++ .../transport/asio/asio_session_manager.cpp | 18 ++++ .../transport/asio/asio_session_manager.h | 8 ++ .../transport/session_manager_common.cpp | 8 +- 8 files changed, 275 insertions(+), 20 deletions(-) create mode 100644 jstests/noPassthrough/network/maintenance_port_metrics.js diff --git a/jstests/noPassthrough/network/maintenance_port_metrics.js b/jstests/noPassthrough/network/maintenance_port_metrics.js new file mode 100644 index 00000000000..9f907d42354 --- /dev/null +++ b/jstests/noPassthrough/network/maintenance_port_metrics.js @@ -0,0 +1,86 @@ +/** + * Tests metrics and logging for the maintenance port. + * + * @tags: [ + * featureFlagDedicatedPortForMaintenanceOperations, + * ] + */ +import {describe, before, after, it} from "jstests/libs/mochalite.js"; +import {checkLog} from "src/mongo/shell/check_log.js"; + +describe("Tests metrics and logging for connections via the maintenance port", function () { + const logID = 22943; + + function assertDoesNotContain(conn, id, msg) { + const logs = checkLog.getGlobalLog(conn); + let containsLog = logs.some((log) => { + if (log.search(`"id":${id},`) != -1) { + return log.search(msg) != -1; + } else { + return false; + } + }); + assert.neq(containsLog, true, "Found log line when none should exist" + tojson(msg)); + } + + function assertMaintenancePortConnCountServerStatusMetricMatches(conn, expectedCount) { + let currentCount; + assert.soon( + () => { + let connectionMetrics = assert.commandWorked(conn.adminCommand({serverStatus: 1})).connections; + currentCount = connectionMetrics.maintenance; + return currentCount == expectedCount; + }, + () => { + return ( + "Incorrect number of maintenance port connections: expected " + + expectedCount + + ", but serverStatus() reports " + + currentCount + ); + }, + ); + } + + before(() => { + this.conn = MongoRunner.runMongod({maintenancePort: allocatePort(), bind_ip: "127.0.0.1", useHostname: false}); + this.host = this.conn.hostNoPort; + this.mainPort = this.conn.port; + this.maintenancePort = this.conn.maintenancePort; + }); + + after(() => { + MongoRunner.stopMongod(this.conn); + }); + + it("Check that normal connections don't log or increment the maintenance metrics", () => { + let conn = new Mongo(this.host + ":" + this.mainPort); + + const msg = /"isMaintenance":"true"/; + assertDoesNotContain(conn, logID, msg); + assertMaintenancePortConnCountServerStatusMetricMatches(conn, 0); + + conn.close(); + }); + + it("Check that maintenance port connections log and increment stats", () => { + let mainConn = new Mongo(this.host + ":" + this.mainPort); + let conns = []; + for (let i = 1; i < 5; i++) { + let newConn = new Mongo(this.host + ":" + this.maintenancePort); + + const msg = /"isMaintenance":"true"/; + assert( + checkLog.checkContainsWithCountJson(mainConn, logID, {"isMaintenance": true}, i), + "Expecting to see " + i + " instances of log " + logID + " with " + msg, + ); + assertMaintenancePortConnCountServerStatusMetricMatches(mainConn, i); + + conns.push(newConn); + } + conns.forEach((conn) => { + conn.close(); + }); + assertMaintenancePortConnCountServerStatusMetricMatches(mainConn, 0); + }); +}); diff --git a/jstests/noPassthrough/query/log_find_getmore.js b/jstests/noPassthrough/query/log_find_getmore.js index 2128b752d5c..90986960ccd 100644 --- a/jstests/noPassthrough/query/log_find_getmore.js +++ b/jstests/noPassthrough/query/log_find_getmore.js @@ -2,6 +2,7 @@ * Confirms that the log output for find and getMore are in the expected format. * @tags: [requires_profiling] */ +import {FeatureFlagUtil} from "jstests/libs/feature_flag_util.js"; import {getLatestProfilerEntry} from "jstests/libs/profiler.js"; function assertLogLineContains(conn, parts) { @@ -57,12 +58,21 @@ cursor.next(); // Perform initial query and retrieve first document in batch. let cursorid = getLatestProfilerEntry(testDB).cursorid; -let logLine = [ - '"msg":"Slow query","attr":{"type":"command",', - '"isFromUserConnection":true,"ns":"log_getmore.test","collectionType":"normal","appName":"MongoDB Shell",', - '"command":{"find":"test","filter":{"a":{"$gt":0}},"skip":1,"batchSize":5,"limit":10,"singleBatch":false,"sort":{"a":1},"hint":{"a":1}', - '"planCacheShapeHash":', -]; +let maintenancePortFFEnabled = FeatureFlagUtil.isPresentAndEnabled(testDB, "DedicatedPortForMaintenanceOperations"); + +let logLine = maintenancePortFFEnabled + ? [ + '"msg":"Slow query","attr":{"type":"command",', + '"isFromUserConnection":true,"isFromMaintenancePortConnection":false,"ns":"log_getmore.test","collectionType":"normal","appName":"MongoDB Shell",', + '"command":{"find":"test","filter":{"a":{"$gt":0}},"skip":1,"batchSize":5,"limit":10,"singleBatch":false,"sort":{"a":1},"hint":{"a":1}', + '"planCacheShapeHash":', + ] + : [ + '"msg":"Slow query","attr":{"type":"command",', + '"isFromUserConnection":true,"ns":"log_getmore.test","collectionType":"normal","appName":"MongoDB Shell",', + '"command":{"find":"test","filter":{"a":{"$gt":0}},"skip":1,"batchSize":5,"limit":10,"singleBatch":false,"sort":{"a":1},"hint":{"a":1}', + '"planCacheShapeHash":', + ]; // Check the logs to verify that find appears as above. assertLogLineContains(conn, logLine); @@ -83,13 +93,21 @@ function cursorIdToString(cursorId) { return cursorIdString.substring('NumberLong("'.length, cursorIdString.length - '")'.length); } -logLine = [ - '"msg":"Slow query"', - '"attr":{"type":"command","isFromUserConnection":true,"ns":"log_getmore.test","collectionType":"normal","appName":"MongoDB Shell"', - `"command":{"getMore":${cursorIdToString(cursorid)},"collection":"test","batchSize":5,`, - '"originatingCommand":{"find":"test","filter":{"a":{"$gt":0}},"skip":1,"batchSize":5,"limit":10,"singleBatch":false,"sort":{"a":1},"hint":{"a":1}', - '"planCacheShapeHash":', -]; +logLine = maintenancePortFFEnabled + ? [ + '"msg":"Slow query"', + '"attr":{"type":"command","isFromUserConnection":true,"isFromMaintenancePortConnection":false,"ns":"log_getmore.test","collectionType":"normal","appName":"MongoDB Shell"', + `"command":{"getMore":${cursorIdToString(cursorid)},"collection":"test","batchSize":5,`, + '"originatingCommand":{"find":"test","filter":{"a":{"$gt":0}},"skip":1,"batchSize":5,"limit":10,"singleBatch":false,"sort":{"a":1},"hint":{"a":1}', + '"planCacheShapeHash":', + ] + : [ + '"msg":"Slow query"', + '"attr":{"type":"command","isFromUserConnection":true,"ns":"log_getmore.test","collectionType":"normal","appName":"MongoDB Shell"', + `"command":{"getMore":${cursorIdToString(cursorid)},"collection":"test","batchSize":5,`, + '"originatingCommand":{"find":"test","filter":{"a":{"$gt":0}},"skip":1,"batchSize":5,"limit":10,"singleBatch":false,"sort":{"a":1},"hint":{"a":1}', + '"planCacheShapeHash":', + ]; assertLogLineContains(conn, logLine); @@ -99,12 +117,19 @@ cursorid = getLatestProfilerEntry(testDB).cursorid; assert.eq(cursor.itcount(), 10); -logLine = [ - '"msg":"Slow query"', - '"attr":{"type":"command","isFromUserConnection":true,"ns":"log_getmore.test","collectionType":"normal","appName":"MongoDB Shell",', - `"command":{"getMore":${cursorIdToString(cursorid)},"collection":"test"`, - '"originatingCommand":{"aggregate":"test","pipeline":[{"$match":{"a":{"$gt":0}}}],"cursor":{"batchSize":0},"hint":{"a":1}', -]; +logLine = maintenancePortFFEnabled + ? [ + '"msg":"Slow query"', + '"attr":{"type":"command","isFromUserConnection":true,"isFromMaintenancePortConnection":false,"ns":"log_getmore.test","collectionType":"normal","appName":"MongoDB Shell",', + `"command":{"getMore":${cursorIdToString(cursorid)},"collection":"test"`, + '"originatingCommand":{"aggregate":"test","pipeline":[{"$match":{"a":{"$gt":0}}}],"cursor":{"batchSize":0},"hint":{"a":1}', + ] + : [ + '"msg":"Slow query"', + '"attr":{"type":"command","isFromUserConnection":true,"ns":"log_getmore.test","collectionType":"normal","appName":"MongoDB Shell",', + `"command":{"getMore":${cursorIdToString(cursorid)},"collection":"test"`, + '"originatingCommand":{"aggregate":"test","pipeline":[{"$match":{"a":{"$gt":0}}}],"cursor":{"batchSize":0},"hint":{"a":1}', + ]; assertLogLineContains(conn, logLine); diff --git a/src/mongo/db/curop.cpp b/src/mongo/db/curop.cpp index 1e93f4df90e..24bed925bd0 100644 --- a/src/mongo/db/curop.cpp +++ b/src/mongo/db/curop.cpp @@ -337,6 +337,11 @@ void CurOp::reportCurrentOpForClient(const boost::intrusive_ptrappendBool("isFromUserConnection", client->isFromUserConnection()); + if (gFeatureFlagDedicatedPortForMaintenanceOperations.isEnabled()) { + infoBuilder->appendBool("isFromMaintenancePortConnection", + client->session() && + client->session()->isConnectedToMaintenancePort()); + } if (transport::ServiceExecutorContext::get(client)) { infoBuilder->append("threaded"_sd, true); diff --git a/src/mongo/db/curop_test.cpp b/src/mongo/db/curop_test.cpp index 6e0a455c243..c945ff30860 100644 --- a/src/mongo/db/curop_test.cpp +++ b/src/mongo/db/curop_test.cpp @@ -36,7 +36,9 @@ #include "mongo/db/operation_context_options_gen.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/query/query_test_service_context.h" +#include "mongo/db/server_feature_flags_gen.h" #include "mongo/idl/server_parameter_test_controller.h" +#include "mongo/transport/mock_session.h" #include "mongo/transport/transport_layer_mock.h" #include "mongo/unittest/death_test.h" #include "mongo/unittest/unittest.h" @@ -871,6 +873,99 @@ TEST(CurOpTest, ShouldReportIsFromUserConnection) { ASSERT_TRUE(bsonObjUserConn.getField("isFromUserConnection").Bool()); } +class MockMaintenanceSession : public transport::MockSession { +public: + explicit MockMaintenanceSession(transport::TransportLayer* tl) : MockSession(tl) {} + + bool isConnectedToMaintenancePort() const override { + return true; + } +}; + +TEST(CurOpTest, ShouldNotReportIsFromMaintenancePortConnectionWhenFFDisabled) { + gFeatureFlagDedicatedPortForMaintenanceOperations.setForServerParameter(false); + + QueryTestServiceContext serviceContext; + auto opCtx = serviceContext.makeOperationContext(); + auto client = serviceContext.getClient(); + + // Mock a client with a user connection. + transport::TransportLayerMock transportLayer; + transportLayer.createSessionHook = [](transport::TransportLayer* tl) { + return std::make_shared(tl); + }; + auto clientMaintenanceConn = serviceContext.getServiceContext()->getService()->makeClient( + "maintenanceConn", transportLayer.createSession()); + + auto curop = CurOp::get(*opCtx); + + BSONObjBuilder curOpObj; + BSONObjBuilder curOpObjMaintenanceConn; + { + stdx::lock_guard lk(*opCtx->getClient()); + auto nss = NamespaceString::createNamespaceString_forTest("db", "coll"); + + // Serialization Context on expression context should be non-empty in + // reportCurrentOpForClient. + auto sc = SerializationContext(SerializationContext::Source::Command, + SerializationContext::CallerType::Reply, + SerializationContext::Prefix::ExcludePrefix); + auto expCtx = make_intrusive(opCtx.get(), nss, sc); + + curop->reportCurrentOpForClient(expCtx, client, false, &curOpObj); + curop->reportCurrentOpForClient( + expCtx, clientMaintenanceConn.get(), false, &curOpObjMaintenanceConn); + } + auto bsonObj = curOpObj.done(); + auto bsonObjMaintenanceConn = curOpObjMaintenanceConn.done(); + + ASSERT_FALSE(bsonObj.hasField("isFromMaintenancePortConnection")); + ASSERT_FALSE(bsonObjMaintenanceConn.hasField("isFromMaintenancePortConnection")); +} + +TEST(CurOpTest, ShouldReportIsFromMaintenancePortConnection) { + gFeatureFlagDedicatedPortForMaintenanceOperations.setForServerParameter(true); + + QueryTestServiceContext serviceContext; + auto opCtx = serviceContext.makeOperationContext(); + auto client = serviceContext.getClient(); + + // Mock a client with a user connection. + transport::TransportLayerMock transportLayer; + transportLayer.createSessionHook = [](transport::TransportLayer* tl) { + return std::make_shared(tl); + }; + auto clientMaintenanceConn = serviceContext.getServiceContext()->getService()->makeClient( + "maintenanceConn", transportLayer.createSession()); + + auto curop = CurOp::get(*opCtx); + + BSONObjBuilder curOpObj; + BSONObjBuilder curOpObjMaintenanceConn; + { + stdx::lock_guard lk(*opCtx->getClient()); + auto nss = NamespaceString::createNamespaceString_forTest("db", "coll"); + + // Serialization Context on expression context should be non-empty in + // reportCurrentOpForClient. + auto sc = SerializationContext(SerializationContext::Source::Command, + SerializationContext::CallerType::Reply, + SerializationContext::Prefix::ExcludePrefix); + auto expCtx = make_intrusive(opCtx.get(), nss, sc); + + curop->reportCurrentOpForClient(expCtx, client, false, &curOpObj); + curop->reportCurrentOpForClient( + expCtx, clientMaintenanceConn.get(), false, &curOpObjMaintenanceConn); + } + auto bsonObj = curOpObj.done(); + auto bsonObjMaintenanceConn = curOpObjMaintenanceConn.done(); + + ASSERT_TRUE(bsonObj.hasField("isFromMaintenancePortConnection")); + ASSERT_TRUE(bsonObjMaintenanceConn.hasField("isFromMaintenancePortConnection")); + ASSERT_FALSE(bsonObj.getField("isFromMaintenancePortConnection").Bool()); + ASSERT_TRUE(bsonObjMaintenanceConn.getField("isFromMaintenancePortConnection").Bool()); +} + TEST(CurOpTest, ElapsedTimeReflectsTickSource) { QueryTestServiceContext serviceContext; diff --git a/src/mongo/db/op_debug.cpp b/src/mongo/db/op_debug.cpp index 0a34692abdc..a8cded2128e 100644 --- a/src/mongo/db/op_debug.cpp +++ b/src/mongo/db/op_debug.cpp @@ -40,6 +40,7 @@ #include "mongo/db/query/plan_summary_stats.h" #include "mongo/db/repl/local_oplog_info.h" #include "mongo/db/repl/read_concern_args.h" +#include "mongo/db/server_feature_flags_gen.h" #include "mongo/db/shard_role/shard_catalog/raw_data_operation.h" #include "mongo/logv2/log.h" #include "mongo/rpc/metadata/client_metadata.h" @@ -197,6 +198,11 @@ void OpDebug::report(OperationContext* opCtx, } pAttrs->add("isFromUserConnection", client && client->isFromUserConnection()); + if (gFeatureFlagDedicatedPortForMaintenanceOperations.isEnabled()) { + pAttrs->add("isFromMaintenancePortConnection", + client && client->session() && + client->session()->isConnectedToMaintenancePort()); + } pAttrs->addDeepCopy("ns", toStringForLogging(curop.getNSS())); pAttrs->addDeepCopy("collectionType", getCollectionType(opCtx, curop.getNSS())); @@ -561,6 +567,12 @@ void OpDebug::append(OperationContext* opCtx, b.append("ns", curop.getNS()); + if (gFeatureFlagDedicatedPortForMaintenanceOperations.isEnabled()) { + b.append("isFromMaintenancePortConnection", + opCtx->getClient() && opCtx->getClient()->session() && + opCtx->getClient()->session()->isConnectedToMaintenancePort()); + } + if (!omitCommand) { curop_bson_helpers::appendObjectTruncatingAsNecessary( "command", diff --git a/src/mongo/transport/asio/asio_session_manager.cpp b/src/mongo/transport/asio/asio_session_manager.cpp index 7c5d2533c9e..0f31de9e26d 100644 --- a/src/mongo/transport/asio/asio_session_manager.cpp +++ b/src/mongo/transport/asio/asio_session_manager.cpp @@ -30,6 +30,7 @@ #include "mongo/transport/asio/asio_session_manager.h" #include "mongo/db/commands/server_status/server_status.h" +#include "mongo/db/server_feature_flags_gen.h" #include "mongo/transport/hello_metrics.h" #include "mongo/transport/service_executor.h" #include "mongo/transport/service_executor_reserved.h" @@ -114,6 +115,9 @@ void AsioSessionManager::appendStats(BSONObjBuilder* bob) const { } bob->append("loadBalanced", _loadBalancedConnections.get()); + if (gFeatureFlagDedicatedPortForMaintenanceOperations.isEnabled()) { + bob->append("maintenance", _maintenancePortConnections.get()); + } } void AsioSessionManager::incrementLBConnections() { @@ -124,6 +128,14 @@ void AsioSessionManager::decrementLBConnections() { _loadBalancedConnections.decrement(); } +void AsioSessionManager::incrementMaintenanceConnections() { + _maintenancePortConnections.increment(); +} + +void AsioSessionManager::decrementMaintenanceConnections() { + _maintenancePortConnections.decrement(); +} + /** * In practice, we will never pass "isLoadBalancerPeer" on connect, * because the client hasn't performed a "hello: {loadBalancer: true} yet. @@ -143,6 +155,9 @@ void AsioSessionManager::onClientConnect(Client* client) { if (session && session->isLoadBalancerPeer()) { incrementLBConnections(); } + if (session && session->isConnectedToMaintenancePort()) { + incrementMaintenanceConnections(); + } } void AsioSessionManager::onClientDisconnect(Client* client) { @@ -150,6 +165,9 @@ void AsioSessionManager::onClientDisconnect(Client* client) { if (session && session->isLoadBalancerPeer()) { decrementLBConnections(); } + if (session && session->isConnectedToMaintenancePort()) { + decrementMaintenanceConnections(); + } } } // namespace mongo::transport diff --git a/src/mongo/transport/asio/asio_session_manager.h b/src/mongo/transport/asio/asio_session_manager.h index 0b211d8aa78..608bec108d7 100644 --- a/src/mongo/transport/asio/asio_session_manager.h +++ b/src/mongo/transport/asio/asio_session_manager.h @@ -50,6 +50,13 @@ public: void incrementLBConnections(); void decrementLBConnections(); + /** + * Increments and decrements the count of total maintenance port connections. + * Currently only implemented in asio_session_manager. + */ + void incrementMaintenanceConnections(); + void decrementMaintenanceConnections(); + protected: std::string getClientThreadName(const Session&) const override; void configureServiceExecutorContext(Client* client, bool isPrivilegedSession) const override; @@ -58,6 +65,7 @@ protected: private: Counter64 _loadBalancedConnections; + Counter64 _maintenancePortConnections; }; } // namespace mongo::transport diff --git a/src/mongo/transport/session_manager_common.cpp b/src/mongo/transport/session_manager_common.cpp index 8729e1ffaa0..5226c26432a 100644 --- a/src/mongo/transport/session_manager_common.cpp +++ b/src/mongo/transport/session_manager_common.cpp @@ -37,6 +37,7 @@ #include "mongo/db/auth/restriction_environment.h" #include "mongo/db/multitenancy_gen.h" +#include "mongo/db/server_feature_flags_gen.h" #include "mongo/db/server_options.h" #include "mongo/logv2/log.h" #include "mongo/platform/atomic_word.h" @@ -65,7 +66,8 @@ struct ClientSummary { remote(c->session()->remote()), sourceClient(c->session()->getSourceRemoteEndpoint()), id(c->session()->id()), - isLoadBalanced(c->session()->isConnectedToLoadBalancerPort()) {} + isLoadBalanced(c->session()->isConnectedToLoadBalancerPort()), + isMaintenance(c->session()->isConnectedToMaintenancePort()) {} friend logv2::DynamicAttributes logAttrs(const ClientSummary& m) { logv2::DynamicAttributes attrs; @@ -74,6 +76,9 @@ struct ClientSummary { if (m.isLoadBalanced) { attrs.add("sourceClient", m.sourceClient); } + if (gFeatureFlagDedicatedPortForMaintenanceOperations.isEnabled()) { + attrs.add("isMaintenance", m.isMaintenance); + } attrs.add("uuid", m.uuid); attrs.add("connectionId", m.id); @@ -85,6 +90,7 @@ struct ClientSummary { HostAndPort sourceClient; SessionId id; bool isLoadBalanced; + bool isMaintenance; }; bool quiet() {