SERVER-69148 Added looser python linting back to mongo pretty printers

.pylintrc

@@ -31,7 +31,52 @@ variable-rgx=[a-z_][a-z0-9_]{1,50}$
 # R0801 - duplicate-code - See PM-1380
 # E0611 - no-name-in-module
 
-disable=bad-continuation,consider-using-sys-exit,deprecated-method,deprecated-module,duplicate-code,exec-used,fixme,import-error,import-outside-toplevel,line-too-long,locally-disabled,missing-class-docstring,missing-docstring,missing-docstring,missing-function-docstring,missing-module-docstring,no-else-break,no-else-continue,no-else-raise,no-else-return,no-member,no-name-in-module,no-self-use,raise-missing-from,redefined-variable-type,super-with-arguments,too-few-public-methods,too-many-arguments,too-many-branches,too-many-function-args,too-many-instance-attributes,too-many-lines,too-many-locals,too-many-public-methods,too-many-return-statements,too-many-statements,unnecessary-comprehension,unnecessary-pass,unused-import,useless-object-inheritance
+disable=
+    bad-continuation,
+    bare-except,
+    broad-except,
+    consider-using-sys-exit,
+    deprecated-method,
+    deprecated-module,
+    duplicate-code,
+    exec-used,
+    fixme,
+    import-error,
+    import-outside-toplevel,
+    line-too-long,
+    locally-disabled,
+    missing-class-docstring,
+    missing-docstring,
+    missing-function-docstring,
+    missing-module-docstring,
+    no-else-break,
+    no-else-continue,
+    no-else-raise,
+    no-else-return,
+    no-member,
+    no-name-in-module,
+    no-self-use,
+    raise-missing-from,
+    redefined-variable-type,
+    subprocess-run-check,
+    super-with-arguments,
+    too-few-public-methods,
+    too-many-arguments,
+    too-many-branches,
+    too-many-function-args,
+    too-many-instance-attributes,
+    too-many-lines,
+    too-many-locals,
+    too-many-public-methods,
+    too-many-return-statements,
+    too-many-statements,
+    unidiomatic-typecheck,
+    unnecessary-comprehension,
+    unnecessary-pass,
+    unused-import,
+    unused-argument,
+    useless-object-inheritance,
+    wrong-import-order
 enable=useless-suppression
 
 [IMPORTS]

buildscripts/gdb/mongo.py

@@ -8,10 +8,15 @@ import sys
 import glob
 import subprocess
 import warnings
-import textwrap
+from pathlib import Path
 
 import gdb
 
+if not gdb:
+    sys.path.insert(0, str(Path(os.path.abspath(__file__)).parent.parent.parent))
+    from buildscripts.gdb.mongo_printers import absl_get_nodes, get_unique_ptr
+
+
 def detect_toolchain(progspace):
 
     readelf_bin = '/opt/mongodbtoolchain/v4/bin/readelf'
@@ -48,7 +53,7 @@ def detect_toolchain(progspace):
     if gcc_version:
         if int(gcc_version.split('.')[0]) == 8:
             toolchain_ver = 'v3'
-        elif int(gcc_version.split('.')[0])  == 11:
+        elif int(gcc_version.split('.')[0]) == 11:
             toolchain_ver = 'v4'
 
     if not toolchain_ver:
@@ -65,27 +70,35 @@ STDERR:
 -----------------
 Assuming {toolchain_ver} as a default, this could cause issues with the printers.""")
 
-    pp = glob.glob(f"/opt/mongodbtoolchain/{toolchain_ver}/share/gcc-*/python/libstdcxx/v6/printers.py")
+    pp = glob.glob(
+        f"/opt/mongodbtoolchain/{toolchain_ver}/share/gcc-*/python/libstdcxx/v6/printers.py")
     printers = pp[0]
     return os.path.dirname(os.path.dirname(os.path.dirname(printers)))
 
+
 stdcxx_printer_toolchain_paths = dict()
+
+
 def load_libstdcxx_printers(progspace):
     if progspace not in stdcxx_printer_toolchain_paths:
         stdcxx_printer_toolchain_paths[progspace] = detect_toolchain(progspace)
         try:
             sys.path.insert(0, stdcxx_printer_toolchain_paths[progspace])
+            global stdlib_printers  # pylint: disable=invalid-name,global-variable-not-assigned
             from libstdcxx.v6 import register_libstdcxx_printers
             from libstdcxx.v6 import printers as stdlib_printers
             register_libstdcxx_printers(progspace)
-            print(f"Loaded libstdc++ pretty printers from '{stdcxx_printer_toolchain_paths[progspace]}'")
+            print(
+                f"Loaded libstdc++ pretty printers from '{stdcxx_printer_toolchain_paths[progspace]}'"
+            )
         except Exception as exc:
-            print(f"Failed to load the libstdc++ pretty printers from {stdcxx_printer_toolchain_paths[progspace]}: {exc}")
+            print(
+                f"Failed to load the libstdc++ pretty printers from {stdcxx_printer_toolchain_paths[progspace]}: {exc}"
+            )
+
 
 def on_new_object_file(objfile) -> None:
-    """Import the libstdc++ GDB pretty printers when either the `attach <pid>` or
-    `core-file <pathname>` commands are run in GDB.
-    """
+    """Import the libstdc++ GDB pretty printers when either the `attach <pid>` or `core-file <pathname>` commands are run in GDB."""
     progspace = objfile.new_objfile.progspace
     if progspace.filename is None:
         # The `attach` command would have filled in the filename so we only need to check if
@@ -98,9 +111,9 @@ def on_new_object_file(objfile) -> None:
                 " and reloading the core dump with the `core-file` command")
         return
 
-
     load_libstdcxx_printers(objfile.new_objfile.progspace)
 
+
 if gdb.selected_inferior().progspace.filename:
     load_libstdcxx_printers(gdb.selected_inferior().progspace)
 else:
@@ -186,7 +199,7 @@ def get_session_kv_pairs():
     session_catalog = get_session_catalog()
     if session_catalog is None:
         return list()
-    return list(absl_get_nodes(session_catalog["_sessions"]))  # pylint: disable=undefined-variable
+    return list(absl_get_nodes(session_catalog["_sessions"]))
 
 
 def get_wt_session(recovery_unit, recovery_unit_impl_type):
@@ -200,11 +213,11 @@ def get_wt_session(recovery_unit, recovery_unit_impl_type):
         return None
     if not recovery_unit:
         return None
-    wt_session_handle = get_unique_ptr(recovery_unit["_session"])  # pylint: disable=undefined-variable
+    wt_session_handle = get_unique_ptr(recovery_unit["_session"])
     if not wt_session_handle.dereference().address:
         return None
     wt_session = wt_session_handle.dereference().cast(
-        gdb.lookup_type("mongo::WiredTigerSession"))["_session"]  # pylint: disable=undefined-variable
+        gdb.lookup_type("mongo::WiredTigerSession"))["_session"]
     return wt_session
 
 
@@ -236,7 +249,7 @@ def get_decorations(obj):
         type_name = type_name[0:type_name.rindex(">")]
         type_name = type_name[type_name.index("constructAt<"):].replace("constructAt<", "")
         # get_unique_ptr should be loaded from 'mongo_printers.py'.
-        decoration_data = get_unique_ptr(decorable["_decorations"]["_decorationData"])  # pylint: disable=undefined-variable
+        decoration_data = get_unique_ptr(decorable["_decorations"]["_decorationData"])
 
         if type_name.endswith('*'):
             type_name = type_name[0:len(type_name) - 1]
@@ -337,7 +350,7 @@ class GetMongoDecoration(gdb.Command):
         """Initialize GetMongoDecoration."""
         RegisterMongoCommand.register(self, "mongo-decoration", gdb.COMMAND_DATA)
 
-    def invoke(self, args, _from_tty):  # pylint: disable=unused-argument
+    def invoke(self, args, _from_tty):
         """Invoke GetMongoDecoration."""
         argarr = args.split(" ")
         if len(argarr) < 2:
@@ -375,7 +388,7 @@ class DumpMongoDSessionCatalog(gdb.Command):
         """Initialize DumpMongoDSessionCatalog."""
         RegisterMongoCommand.register(self, "mongod-dump-sessions", gdb.COMMAND_DATA)
 
-    def invoke(self, args, _from_tty):  # pylint: disable=unused-argument
+    def invoke(self, args, _from_tty):
         """Invoke DumpMongoDSessionCatalog."""
         # See if a particular session id was specified.
         argarr = args.split(" ")
@@ -402,9 +415,9 @@ class DumpMongoDSessionCatalog(gdb.Command):
 
         for session_kv in session_kv_pairs:
             # The Session objects are stored inside the SessionRuntimeInfo object.
-            session_runtime_info = get_unique_ptr(session_kv['second']).dereference()  # pylint: disable=undefined-variable
+            session_runtime_info = get_unique_ptr(session_kv['second']).dereference()
             parent_session = session_runtime_info['parentSession']
-            child_sessions = absl_get_nodes(session_runtime_info['childSessions'])  # pylint: disable=undefined-variable
+            child_sessions = absl_get_nodes(session_runtime_info['childSessions'])
             lsid = str(parent_session['_sessionId']['_id'])
 
             # If we are only interested in a specific session, then we print out the entire Session
@@ -487,7 +500,7 @@ class DumpMongoDSessionCatalog(gdb.Command):
             # the need for special unpacking.
             val = get_boost_optional(txn_part_observable_state['txnResourceStash'])
             if val:
-                locker_addr = get_unique_ptr(val["_locker"])  # pylint: disable=undefined-variable
+                locker_addr = get_unique_ptr(val["_locker"])
                 locker_obj = locker_addr.dereference().cast(gdb.lookup_type("mongo::LockerImpl"))
                 print('txnResourceStash._locker', "@", locker_addr)
                 print("txnResourceStash._locker._id", "=", locker_obj["_id"])
@@ -513,7 +526,7 @@ class DumpMongoDBMutexes(gdb.Command):
         print("Dumping mutex info for all Clients")
 
         service_context = get_global_service_context()
-        client_set = absl_get_nodes(service_context["_clients"])  # pylint: disable=undefined-variable
+        client_set = absl_get_nodes(service_context["_clients"])
         for client_handle in client_set:
             client = client_handle.dereference().dereference()
             decoration_info = get_decoration(client, "DiagnosticInfoHandle")
@@ -523,7 +536,7 @@ class DumpMongoDBMutexes(gdb.Command):
             diagnostic_info_list = diagnostic_info_handle["list"]
 
             # Use the STL pretty-printer to iterate over the list
-            printer = stdlib_printers.StdForwardListPrinter(
+            printer = stdlib_printers.StdForwardListPrinter(  # pylint: disable=undefined-variable
                 str(diagnostic_info_list.type), diagnostic_info_list)
 
             # Prepare structured output doc
@@ -573,8 +586,9 @@ class MongoDBDumpLocks(gdb.Command):
             # Note that output will go to mongod's standard output, not the debugger output window
             # Do not call mongo::getGlobalLockManager() due to the compiler optimizing this function in a very weird way
             # See SERVER-72816 for more context
-            gdb.execute("call mongo::LockManager::get((mongo::ServiceContext*) mongo::getGlobalServiceContext())->dump()", from_tty=False,
-                        to_string=False)
+            gdb.execute(
+                "call mongo::LockManager::get((mongo::ServiceContext*) mongo::getGlobalServiceContext())->dump()",
+                from_tty=False, to_string=False)
         except gdb.error as gdberr:
             print("Ignoring error '%s' in dump_mongod_locks" % str(gdberr))
 
@@ -616,7 +630,7 @@ class MongoDBDumpRecoveryUnits(gdb.Command):
 
         # Dump active recovery unit info for each client in a mongod process
         service_context = get_global_service_context()
-        client_set = absl_get_nodes(service_context["_clients"])  # pylint: disable=undefined-variable
+        client_set = absl_get_nodes(service_context["_clients"])
 
         for client_handle in client_set:
             client = client_handle.dereference().dereference()
@@ -630,7 +644,7 @@ class MongoDBDumpRecoveryUnits(gdb.Command):
             recovery_unit = None
             if operation_context_handle:
                 operation_context = operation_context_handle.dereference()
-                recovery_unit_handle = get_unique_ptr(operation_context["_recoveryUnit"])  # pylint: disable=undefined-variable
+                recovery_unit_handle = get_unique_ptr(operation_context["_recoveryUnit"])
                 # By default, cast the recovery unit as "mongo::WiredTigerRecoveryUnit"
                 recovery_unit = recovery_unit_handle.dereference().cast(
                     gdb.lookup_type(recovery_unit_impl_type))
@@ -646,9 +660,9 @@ class MongoDBDumpRecoveryUnits(gdb.Command):
         # Dump stashed recovery unit info for each session in a mongod process
         for session_kv in get_session_kv_pairs():
             # The Session objects are stored inside the SessionRuntimeInfo object.
-            session_runtime_info = get_unique_ptr(session_kv['second']).dereference()  # pylint: disable=undefined-variable
+            session_runtime_info = get_unique_ptr(session_kv['second']).dereference()
             parent_session = session_runtime_info['parentSession']
-            child_sessions = absl_get_nodes(session_runtime_info['childSessions'])  # pylint: disable=undefined-variable
+            child_sessions = absl_get_nodes(session_runtime_info['childSessions'])
 
             MongoDBDumpRecoveryUnits.dump_session(parent_session, recovery_unit_impl_type)
             for child_session_kv in child_sessions:
@@ -675,7 +689,7 @@ class MongoDBDumpRecoveryUnits(gdb.Command):
                 txn_participant_observable_state["txnResourceStash"])
             if txn_resource_stash:
                 output_doc["txnResourceStash"] = str(txn_resource_stash.address)
-                recovery_unit_handle = get_unique_ptr(txn_resource_stash["_recoveryUnit"])  # pylint: disable=undefined-variable
+                recovery_unit_handle = get_unique_ptr(txn_resource_stash["_recoveryUnit"])
                 # By default, cast the recovery unit as "mongo::WiredTigerRecoveryUnit"
                 recovery_unit = recovery_unit_handle.dereference().cast(
                     gdb.lookup_type(recovery_unit_impl_type))

buildscripts/gdb/mongo_lock.py

@@ -2,10 +2,15 @@
 
 import re
 import sys
-
+import os
+from pathlib import Path
 import gdb
 import gdb.printing
 
+if not gdb:
+    sys.path.insert(0, str(Path(os.path.abspath(__file__)).parent.parent.parent))
+    from buildscripts.gdb.mongo import get_current_thread_name, get_thread_id, RegisterMongoCommand
+
 if sys.version_info[0] < 3:
     raise gdb.GdbError(
         "MongoDB gdb extensions only support Python 3. Your GDB was compiled against Python 2")
@@ -372,8 +377,8 @@ def get_threads_info():
             # PTID is a tuple: Process ID (PID), Lightweight Process ID (LWPID), Thread ID (TID)
             (_, lwpid, _) = thread.ptid
             thread_num = thread.num
-            thread_name = get_current_thread_name()  # pylint: disable=undefined-variable
-            thread_id = get_thread_id()  # pylint: disable=undefined-variable
+            thread_name = get_current_thread_name()
+            thread_id = get_thread_id()
             if not thread_id:
                 print("Unable to retrieve thread_info for thread %d" % thread_num)
                 continue
@@ -389,8 +394,7 @@ class MongoDBShowLocks(gdb.Command):
 
     def __init__(self):
         """Initialize MongoDBShowLocks."""
-        RegisterMongoCommand.register(  # pylint: disable=undefined-variable
-            self, "mongodb-show-locks", gdb.COMMAND_DATA)
+        RegisterMongoCommand.register(self, "mongodb-show-locks", gdb.COMMAND_DATA)
 
     def invoke(self, *_):
         """Invoke mongodb_show_locks."""
@@ -414,8 +418,7 @@ class MongoDBWaitsForGraph(gdb.Command):
 
     def __init__(self):
         """Initialize MongoDBWaitsForGraph."""
-        RegisterMongoCommand.register(  # pylint: disable=undefined-variable
-            self, "mongodb-waitsfor-graph", gdb.COMMAND_DATA)
+        RegisterMongoCommand.register(self, "mongodb-waitsfor-graph", gdb.COMMAND_DATA)
 
     def invoke(self, arg, *_):
         """Invoke mongodb_waitsfor_graph."""

buildscripts/gdb/mongo_printers.py

@@ -26,6 +26,7 @@ def get_unique_ptr(obj):
     """Read the value of a libstdc++ std::unique_ptr."""
     return obj.cast(gdb.lookup_type('std::_Head_base<0, unsigned char*, false>'))['_M_head_impl']
 
+
 ###################################################################################################
 #
 # Pretty-Printers
@@ -370,7 +371,6 @@ class WtCursorPrinter(object):
         """Initializer."""
         self.val = val
 
-    # pylint: disable=R0201
     def to_string(self):
         """to_string."""
         return None
@@ -405,7 +405,6 @@ class WtSessionImplPrinter(object):
         """Initializer."""
         self.val = val
 
-    # pylint: disable=R0201
     def to_string(self):
         """to_string."""
         return None
@@ -440,7 +439,6 @@ class WtTxnPrinter(object):
         """Initializer."""
         self.val = val
 
-    # pylint: disable=R0201
     def to_string(self):
         """to_string."""
         return None
@@ -668,7 +666,6 @@ class WtUpdateToBsonPrinter(object):
         """DisplayHint."""
         return 'map'
 
-    # pylint: disable=R0201
     def to_string(self):
         """ToString."""
         elems = []
@@ -717,9 +714,11 @@ def read_as_integer(pmem, size):
     # We assume the same platform for the debugger and the debuggee (thus, 'sys.byteorder'). If
     # this becomes a problem look into whether it's possible to determine the byteorder of the
     # inferior.
-    return int.from_bytes( \
-        gdb.selected_inferior().read_memory(pmem, size).tobytes(), \
-        sys.byteorder)
+    return int.from_bytes(
+        gdb.selected_inferior().read_memory(pmem, size).tobytes(),
+        sys.byteorder,
+    )
+
 
 def read_as_integer_signed(pmem, size):
     """Read 'size' bytes at 'pmem' as an integer."""
@@ -729,7 +728,9 @@ def read_as_integer_signed(pmem, size):
     return int.from_bytes(
         gdb.selected_inferior().read_memory(pmem, size).tobytes(),
         sys.byteorder,
-        signed = True)
+        signed=True,
+    )
+
 
 class SbeCodeFragmentPrinter(object):
     """
@@ -765,7 +766,6 @@ class SbeCodeFragmentPrinter(object):
         """Return sbe::vm::CodeFragment for printing."""
         return "%s" % (self.val.type)
 
-    # pylint: disable=R0915
     def children(self):
         """children."""
         yield '_instrs', '{... (to see raw output, run "disable pretty-printer")}'
@@ -957,6 +957,7 @@ class IntervalPrinter(OptimizerTypePrinter):
         """Initialize IntervalPrinter."""
         super().__init__(val, "ExplainGenerator::explainInterval")
 
+
 class IntervalExprPrinter(OptimizerTypePrinter):
     """Pretty-printer for mongo::optimizer::IntervalRequirement::Node."""
 
@@ -1023,61 +1024,63 @@ def register_abt_printers(pp):
         pp.add('ABT::Reference', abt_ref_type, False, ABTPrinter)
     except gdb.error:
         # ABT printer.
-        abt_type_set = ["Blackhole",
-                               "Constant",
-                               "Variable",
-                               "UnaryOp",
-                               "BinaryOp",
-                               "If",
-                               "Let",
-                               "LambdaAbstraction",
-                               "LambdaApplication",
-                               "FunctionCall",
-                               "EvalPath",
-                               "EvalFilter",
-                               "Source",
-                               "PathConstant",
-                               "PathLambda",
-                               "PathIdentity",
-                               "PathDefault",
-                               "PathCompare",
-                               "PathDrop",
-                               "PathKeep",
-                               "PathObj",
-                               "PathArr",
-                               "PathTraverse",
-                               "PathField",
-                               "PathGet",
-                               "PathComposeM",
-                               "PathComposeA",
-                               "ScanNode",
-                               "PhysicalScanNode",
-                               "ValueScanNode",
-                               "CoScanNode",
-                               "IndexScanNode",
-                               "SeekNode",
-                               "MemoLogicalDelegatorNode",
-                               "MemoPhysicalDelegatorNode",
-                               "FilterNode",
-                               "EvaluationNode",
-                               "SargableNode",
-                               "RIDIntersectNode",
-                               "RIDUnionNode",
-                               "BinaryJoinNode",
-                               "HashJoinNode",
-                               "MergeJoinNode",
-                               "SortedMergeNode",
-                               "NestedLoopJoinNode",
-                               "UnionNode",
-                               "GroupByNode",
-                               "UnwindNode",
-                               "UniqueNode",
-                               "CollationNode",
-                               "LimitSkipNode",
-                               "ExchangeNode",
-                               "RootNode",
-                               "References",
-                               "ExpressionBinder"]
+        abt_type_set = [
+            "Blackhole",
+            "Constant",
+            "Variable",
+            "UnaryOp",
+            "BinaryOp",
+            "If",
+            "Let",
+            "LambdaAbstraction",
+            "LambdaApplication",
+            "FunctionCall",
+            "EvalPath",
+            "EvalFilter",
+            "Source",
+            "PathConstant",
+            "PathLambda",
+            "PathIdentity",
+            "PathDefault",
+            "PathCompare",
+            "PathDrop",
+            "PathKeep",
+            "PathObj",
+            "PathArr",
+            "PathTraverse",
+            "PathField",
+            "PathGet",
+            "PathComposeM",
+            "PathComposeA",
+            "ScanNode",
+            "PhysicalScanNode",
+            "ValueScanNode",
+            "CoScanNode",
+            "IndexScanNode",
+            "SeekNode",
+            "MemoLogicalDelegatorNode",
+            "MemoPhysicalDelegatorNode",
+            "FilterNode",
+            "EvaluationNode",
+            "SargableNode",
+            "RIDIntersectNode",
+            "RIDUnionNode",
+            "BinaryJoinNode",
+            "HashJoinNode",
+            "MergeJoinNode",
+            "SortedMergeNode",
+            "NestedLoopJoinNode",
+            "UnionNode",
+            "GroupByNode",
+            "UnwindNode",
+            "UniqueNode",
+            "CollationNode",
+            "LimitSkipNode",
+            "ExchangeNode",
+            "RootNode",
+            "References",
+            "ExpressionBinder",
+        ]
         abt_type = "mongo::optimizer::algebra::PolyValue<"
         for type_name in abt_type_set:
             abt_type += "mongo::optimizer::" + type_name

buildscripts/gdb/wt_dump_table.py

@@ -142,7 +142,6 @@ def dump_insert_list(wt_insert):
 def dump_skip_list(wt_insert_head):
     if not wt_insert_head['head'].address:
         return
-    q = wt_insert_head['head']
     wt_insert = wt_insert_head['head'][0]
     idx = 0
     while True:
@@ -181,7 +180,6 @@ def dump_modified(leaf_page):
 
 
 def dump_disk(leaf_page):
-    leaf_num_entries = int(leaf_page['entries'])
     dbg('in-memory page:', leaf_page)
     dsk = leaf_page['dsk'].dereference()
     if int(dsk.address) == 0:

buildscripts/pylinters.py

@@ -69,7 +69,7 @@ def is_interesting_file(file_name):
     # type: (str) -> bool
     """Return true if this file should be checked."""
     file_denylist = []  # type: List[str]
-    directory_denylist = ["src/third_party", "buildscripts/gdb"]
+    directory_denylist = ["src/third_party"]
     if file_name in file_denylist or file_name.startswith(tuple(directory_denylist)):
         return False
     directory_list = ["buildscripts", "pytests"]