From 88c79d8e8221cb2a73cc324b0bc21e7f3211fa63 Mon Sep 17 00:00:00 2001 From: Ronald Steinke <167128994+rsteinkeX@users.noreply.github.com> Date: Thu, 22 May 2025 10:17:39 -0600 Subject: [PATCH] SERVER-103729 Data generators for NEEDS_REPLACEMENT and USE_REPLACEMENT warnings (#35951) GitOrigin-RevId: c8be2a83078ab368fff0dd4a12777b950f4377e1 --- .github/CODEOWNERS | 5 +++ modules_poc/OWNERS.yml | 11 +++++ modules_poc/mod_scanner.py | 54 ++++++++++++++++++----- modules_poc/needs_replacement.jq | 5 +++ modules_poc/no_longer_public.jq | 5 +++ modules_poc/other_team_use_replacement.jq | 5 +++ modules_poc/util.jq | 22 +++++++++ 7 files changed, 95 insertions(+), 12 deletions(-) create mode 100644 modules_poc/OWNERS.yml create mode 100644 modules_poc/needs_replacement.jq create mode 100644 modules_poc/no_longer_public.jq create mode 100644 modules_poc/other_team_use_replacement.jq create mode 100644 modules_poc/util.jq diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index eb8740d7294..5e1e2f23a39 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1372,6 +1372,11 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot # The following patterns are parsed from ./jstests/with_mongot/OWNERS.yml /jstests/with_mongot/**/* @10gen/query-integration-search @svc-auto-approve-bot +# The following patterns are parsed from ./modules_poc/OWNERS.yml +/modules_poc/**/OWNERS.yml @10gen/server-programmability @svc-auto-approve-bot +/modules_poc/**/*.py @10gen/server-programmability @svc-auto-approve-bot +/modules_poc/**/*.jq @10gen/server-programmability @svc-auto-approve-bot + # The following patterns are parsed from ./rpm/OWNERS.yml /rpm/**/* @10gen/devprod-build @svc-auto-approve-bot diff --git a/modules_poc/OWNERS.yml b/modules_poc/OWNERS.yml new file mode 100644 index 00000000000..861be8c2228 --- /dev/null +++ b/modules_poc/OWNERS.yml @@ -0,0 +1,11 @@ +version: 1.0.0 +filters: + - "OWNERS.yml": + approvers: + - 10gen/server-programmability + - "*.py": + approvers: + - 10gen/server-programmability + - "*.jq": + approvers: + - 10gen/server-programmability diff --git a/modules_poc/mod_scanner.py b/modules_poc/mod_scanner.py index 82164969209..7d5e1ff2fb4 100755 --- a/modules_poc/mod_scanner.py +++ b/modules_poc/mod_scanner.py @@ -263,7 +263,19 @@ class DecoratedCursor(Cursor): DETAIL_REGEX = re.compile(r"(detail|internal)s?$") -def get_visibility(c: DecoratedCursor, scanning_parent=False): +@dataclass +class GetVisibilityResult: + attr: str + alt: str | None + parent: DecoratedCursor | None # only None for UNKNOWN + non_ns_parent: DecoratedCursor | None + + +def get_visibility( + c: DecoratedCursor, scanning_parent=False, last_non_ns_parent=None +) -> GetVisibilityResult: + if c.kind != CursorKind.NAMESPACE: + last_non_ns_parent = c if c.has_attrs(): for child in c.get_children(): if child.kind != CursorKind.ANNOTATE_ATTR: @@ -288,13 +300,13 @@ def get_visibility(c: DecoratedCursor, scanning_parent=False): "file_private", "needs_replacement", ) - return (attr, alt) + return GetVisibilityResult(attr, alt, c, last_non_ns_parent) # Apply high-priority defaults that override parent's visibility if not scanning_parent: # TODO consider making PROTECTED also default to module private if c.access_specifier == AccessSpecifier.PRIVATE: - return ("private", None) + return GetVisibilityResult("private", None, c, last_non_ns_parent) # TODO: Unfortunately these rules are violated on 64 declarations, # so it can't be enabled yet. @@ -317,21 +329,23 @@ def get_visibility(c: DecoratedCursor, scanning_parent=False): # declared public anyway? if 0: # :( if c.spelling.endswith("forTest"): - return "private" + return GetVisibilityResult("private", None, c, last_non_ns_parent) # details and internal namespaces if c.kind == CursorKind.NAMESPACE and DETAIL_REGEX.match(c.spelling): - return "private" + return GetVisibilityResult("private", None, c, last_non_ns_parent) if c.normalized_parent: - parent_vis = get_visibility(c.normalized_parent, scanning_parent=True) + parent_vis = get_visibility( + c.normalized_parent, scanning_parent=True, last_non_ns_parent=last_non_ns_parent + ) else: - parent_vis = ("UNKNOWN", None) # break recursion + parent_vis = GetVisibilityResult("UNKNOWN", None, None, None) # break recursion # Apply low-priority defaults that defer to parent's visibility - if not scanning_parent and parent_vis[0] == "UNKNOWN": + if not scanning_parent and parent_vis.attr == "UNKNOWN": if normpath_for_file(c) in complete_headers: - return ("private", None) + return GetVisibilityResult("private", None, c, last_non_ns_parent) return parent_vis @@ -405,6 +419,18 @@ def teams_for_file(f: ClangFile | str | None): return teams if teams else ["__NO_OWNER__"] +def make_vis_from(c: DecoratedCursor | None): + if not c: + return None + return { + "usr": c.normalized_usr, + "display_name": fully_qualified(c), + "kind": c.kind.name, + "loc": pretty_location(c.location), + "mod": mod_for_file(c.location.file), + } + + @dataclass class Decl: display_name: str @@ -419,6 +445,8 @@ class Decl: spelling: str visibility: str alt: str + vis_from: dict[str, str] + vis_from_non_ns: dict[str, str] sem_par: str lex_par: str used_from: dict[str, set[str]] = dataclasses.field(default_factory=dict, compare=False) @@ -430,7 +458,7 @@ class Decl: def from_cursor(c: Cursor, mod=None): if not isinstance(c, DecoratedCursor): c = DecoratedCursor(c) - vis, alt = get_visibility(c) + vis = get_visibility(c) return Decl( display_name=fully_qualified(c), spelling=c.spelling, @@ -442,8 +470,10 @@ class Decl: kind=c.kind.name, mod=mod or mod_for_file(c.location.file), defined=c.has_definition, - visibility=vis, - alt=alt, + visibility=vis.attr, + alt=vis.alt, + vis_from=make_vis_from(vis.parent), + vis_from_non_ns=make_vis_from(vis.non_ns_parent), sem_par=c.normalized_parent.normalized_usr if c.normalized_parent else None, lex_par=( DecoratedCursor(c.lexical_parent).normalized_usr diff --git a/modules_poc/needs_replacement.jq b/modules_poc/needs_replacement.jq new file mode 100644 index 00000000000..3c92bf645ae --- /dev/null +++ b/modules_poc/needs_replacement.jq @@ -0,0 +1,5 @@ +include "util" {search: "./"}; + +map(. as $decl | select((.visibility | test("^needs_replacement")) and + any(.used_from[]; is_submodule($decl) | not))) | +group_by_visibility_parent_non_ns diff --git a/modules_poc/no_longer_public.jq b/modules_poc/no_longer_public.jq new file mode 100644 index 00000000000..e9d6114905b --- /dev/null +++ b/modules_poc/no_longer_public.jq @@ -0,0 +1,5 @@ +include "util" {search: "./"}; + +map(. as $decl | select((.visibility | test("^(use|needs)_replacement")) and + (any(.used_from[]; is_submodule($decl) | not) | not))) | +group_by_visibility_parent diff --git a/modules_poc/other_team_use_replacement.jq b/modules_poc/other_team_use_replacement.jq new file mode 100644 index 00000000000..cfb38e04478 --- /dev/null +++ b/modules_poc/other_team_use_replacement.jq @@ -0,0 +1,5 @@ +include "util" {search: "./"}; + +map(. as $decl | select((.visibility | test("^use_replacement")) and + any(.used_from[]; (is_submodule($decl) | not) and .mod != "__NONE__"))) | +group_by_visibility_parent_non_ns diff --git a/modules_poc/util.jq b/modules_poc/util.jq new file mode 100644 index 00000000000..9a081b31c6a --- /dev/null +++ b/modules_poc/util.jq @@ -0,0 +1,22 @@ +def is_submodule($d): (.mod == $d.mod) or (.mod | startswith("\($d.mod).")); + +def _group_by_vis_map(vis_from): + map({ + usr: vis_from.usr, + display_name: vis_from.display_name, + mod: .[0].mod, + loc: .[0].loc, + kind: .[0].kind, + used_from: [.[].used_from[]] | + group_by(.mod) | + map({ + mod: .[0].mod, + locs: [.[].locs[]] | map(split("\t") | .[1]) | unique, + }), + }); + +def group_by_visibility_parent_non_ns: + group_by(.vis_from_non_ns.usr) | _group_by_vis_map(.[0].vis_from_non_ns); + +def group_by_visibility_parent: + group_by(.vis_from.usr) | _group_by_vis_map(.[0].vis_from);