diff --git a/jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_sharded_cluster.js b/jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_sharded_cluster.js
index a33482099e4..e626640500c 100644
--- a/jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_sharded_cluster.js
+++ b/jstests/multiVersion/genericSetFCVUsage/upgrade_downgrade_sharded_cluster.js
@@ -167,6 +167,20 @@ for (const oldVersion of [lastLTSFCV, lastContinuousFCV]) {
     jsTest.log('Upgrading binaries to latest version');
     st.upgradeCluster('latest');
 
+    const newNode = st.shard0.rs.add({shardsvr: "", rsConfig: {priority: 0, votes: 0}});
+    st.shard0.rs.reInitiate();
+    st.shard0.rs.awaitSecondaryNodes();
+
+    let shard;
+    assert.soon(
+        () => {
+            shard = st.s.getCollection("config.shards").findOne({_id: st.shard0.shardName});
+            const hosts = MongoURI(shard.host).servers.map(x => x.server);
+            return hosts.includes(newNode.host);
+        },
+        () => `timed out waiting for replica set shard primary to add ${
+            newNode.host} to config.shards entry: ${tojson(shard)}`);
+
     checkClusterAfterBinaryUpgrade();
 
     jsTest.log('Upgrading FCV to ' + latestFCV);
diff --git a/src/mongo/s/client/shard_registry.cpp b/src/mongo/s/client/shard_registry.cpp
index a083fcb75eb..e3af625de8f 100644
--- a/src/mongo/s/client/shard_registry.cpp
+++ b/src/mongo/s/client/shard_registry.cpp
@@ -548,8 +548,11 @@ void ShardRegistry::scheduleReplicaSetUpdateOnConfigServerIfNeeded(
         // Specify the config version in the filter and the update to prevent overwriting a
         // newer connection string when there are concurrent updates.
         auto filter =
-            BSON(ShardType::name() << shard->getId().toString() << ShardType::replSetConfigVersion()
-                                   << BSON("$lt" << replSetConfigVersion));
+            BSON(ShardType::name()
+                 << shard->getId().toString() << "$or"
+                 << BSON_ARRAY(BSON(ShardType::replSetConfigVersion() << BSON("$exists" << false))
+                               << BSON(ShardType::replSetConfigVersion()
+                                       << BSON("$lt" << replSetConfigVersion))));
         auto update = BSON("$set" << BSON(ShardType::host()
                                           << connStr.toString() << ShardType::replSetConfigVersion()
                                           << replSetConfigVersion));