SERVER-106115 define list of allowable unowned files (#37204)

GitOrigin-RevId: 9b570a2f3a15d5c9a3b23a1fd041443616b223fb

.bazelrc

@@ -357,6 +357,8 @@ coverage --fission=no
 
 # code ownership configuration
 common --define codeowners_add_auto_approve_user=True
+common --define codeowners_have_allowed_unowned_files=True
+common --define codeowners_allowed_unowned_files_path=.github/ALLOWED_UNOWNED_FILES.yml
 
 # Don't detect the native toolchain on linux, only use the hermetic toolchains.
 # Opt out of this by passing --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=0 on the command line.

.github/ALLOWED_UNOWNED_FILES.yml

@@ -0,0 +1,4 @@
+version: 1.0.0
+filters:
+  - filter: "/.github/CODEOWNERS"
+    justification: "Generated by all of the individual owners files in the repo."

.github/CODEOWNERS

@@ -27,6 +27,10 @@ sbom.json @10gen/server-security @svc-auto-approve-bot
 MODULE.bazel* @10gen/devprod-build @svc-auto-approve-bot
 WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
 
+# The following patterns are parsed from ./.github/OWNERS.yml
+/.github/ @10gen/server-root-ownership @svc-auto-approve-bot
+/.github/ALLOWED_UNOWNED_FILES.yml @svc-auto-approve-bot alex.neben@mongodb.com
+
 # The following patterns are parsed from ./bazel/OWNERS.yml
 /bazel/**/* @10gen/devprod-build @svc-auto-approve-bot
 
@@ -3077,3 +3081,6 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
 
 # The following patterns are parsed from ./x509/OWNERS.yml
 /x509/**/* @10gen/server-security @svc-auto-approve-bot
+
+# The following lines are added from .github/ALLOWED_UNOWNED_FILES.yml
+/.github/CODEOWNERS

.github/OWNERS.yml

@@ -0,0 +1,8 @@
+version: 2.0.0
+filters:
+  - "*":
+    approvers:
+      - 10gen/server-root-ownership
+  - "/ALLOWED_UNOWNED_FILES.yml":
+    approvers:
+      - alex.neben@mongodb.com

buildscripts/bazel_rules_mongo/codeowners/BUILD.bazel

@@ -26,6 +26,11 @@ py_binary(
             "CODEOWNERS_CHECK_NEW_FILES": "false",
         },
         "//conditions:default": {},
+    }) | select({
+        ":have_allowed_unowned_files": {
+            "ALLOWED_UNOWNED_FILES_PATH": "$(codeowners_allowed_unowned_files_path)",
+        },
+        "//conditions:default": {},
     }),
     main = "codeowners_generate.py",
     visibility = ["//visibility:public"],
@@ -52,3 +57,10 @@ config_setting(
         "codeowners_dont_check_new_files": "True",
     },
 )
+
+config_setting(
+    name = "have_allowed_unowned_files",
+    define_values = {
+        "codeowners_have_allowed_unowned_files": "True",
+    },
+)

buildscripts/bazel_rules_mongo/codeowners/codeowners_generate.py

@@ -149,10 +149,11 @@ def check_new_files(codeowners_binary_path: str, expansions_file: str, branch: s
     print(f"The following new files were detected: {new_files}")
 
     unowned_files = get_unowned_files(codeowners_binary_path)
+    allowed_unowned_files = get_allowed_unowned_files()
 
     unowned_new_files = []
     for file in new_files:
-        if file in unowned_files:
+        if file in unowned_files and f"/{file}" not in allowed_unowned_files:
             unowned_new_files.append(file)
 
     if unowned_new_files:
@@ -185,8 +186,13 @@ def check_orphaned_files(
     temp_codeowners_file.write(previous_codeowners_file_contents)
     temp_codeowners_file.close()
     old_unowned_files = get_unowned_files(codeowners_binary_path, temp_codeowners_file.name)
+    allowed_unowned_files = get_allowed_unowned_files()
 
     unowned_files_difference = current_unowned_files - old_unowned_files
+    for file in list(unowned_files_difference):
+        if f"/{file}" in allowed_unowned_files:
+            unowned_files_difference.remove(file)
+
     if not unowned_files_difference:
         print("No files have lost ownership with these changes.")
         return 0
@@ -219,6 +225,68 @@ def post_generation_checks(
     return status
 
 
+def get_allowed_unowned_files_path() -> Optional[str]:
+    return os.environ.get("ALLOWED_UNOWNED_FILES_PATH", None)
+
+
+@cache
+def get_allowed_unowned_files() -> Set[str]:
+    allowed_unowned_file_path = get_allowed_unowned_files_path()
+    if not allowed_unowned_file_path:
+        return set()
+
+    unowned_files = set()
+
+    with open(allowed_unowned_file_path, "r") as file:
+        contents = yaml.safe_load(file)
+
+        try:
+            assert "version" in contents, f"version field not found in {allowed_unowned_file_path}"
+            assert contents["version"] == "1.0.0", f"unknown version in {allowed_unowned_file_path}"
+            del contents["version"]
+
+            working_directory = os.curdir
+            assert "filters" in contents, f"No filters were found in {allowed_unowned_file_path}"
+            for filter in contents["filters"]:
+                assert "justification" in filter, "all filters need a justification"
+                pattern = filter["filter"]
+                assert pattern.startswith("/"), "All unowned file filters must start with a /"
+                assert "*" not in pattern, "No wildcard patterns allowed in unowned file filters."
+                test_pattern = f"{working_directory}{pattern}"
+                assert os.path.exists(test_pattern), f"Filter was not found: {pattern}"
+                assert not os.path.isdir(
+                    test_pattern
+                ), "No directories are allowed in unowned file filters."
+                assert os.path.isfile(test_pattern), f"No files matched pattern: {pattern}"
+
+                unowned_files.add(pattern)
+        except Exception as ex:
+            print(f"Error occurred while parsing {allowed_unowned_file_path}")
+            print(
+                "For documentation around the file format please read https://github.com/10gen/mongo/blob/master/docs/owners/allowed_unowned_files_format.md"
+            )
+            raise ex
+
+    return unowned_files
+
+
+def add_allowed_unowned_files(output_lines: List[str]) -> None:
+    allowed_unowned_files = get_allowed_unowned_files()
+    if not allowed_unowned_files:
+        return
+
+    allowed_unowned_files_path = get_allowed_unowned_files_path()
+    assert (
+        allowed_unowned_files_path
+    ), "Somehow there were allowed unowned files but a path was not found."
+
+    output_lines.append(f"# The following lines are added from {allowed_unowned_files_path}")
+    for file in allowed_unowned_files:
+        output_lines.append(f"{file}")
+    # adds a newline
+    output_lines.append("")
+
+
 def main():
     # If we are running in bazel, default the directory to the workspace
     default_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY")
@@ -304,6 +372,7 @@ def main():
         files = evergreen_git.get_files_to_lint()
         root_node = build_tree(files)
         process_dir(output_lines, root_node)
+        add_allowed_unowned_files(output_lines)
     except Exception as ex:
         print("An exception was found while generating the CODEOWNERS file.", file=sys.stderr)
         print(

buildscripts/bazel_rules_mongo/codeowners/parsers/owners_v1.py

@@ -51,7 +51,7 @@ class OwnersParserV1:
                         # approver is github username, need to prefix with @
                         owners.add(f"@{owner}")
 
-                NOOWNERS_NAME = "NOOWNERS-DO-NOT-USE-DEPRECATED-2024-07-01"
+                NOOWNERS_NAME = "NOOWNERS"
                 if NOOWNERS_NAME in approvers:
                     assert (
                         len(approvers) == 1

buildscripts/bazel_rules_mongo/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bazel_rules_mongo"
-version = "0.1.8"
+version = "0.1.9"
 description = "Bazel rule we use to ship common code between bazel repos"
 authors = ["Trevor Guidry <trevor.guidry@mongodb.com>"]
 readme = "README.md"

docs/owners/allowed_unowned_files_format.md

@@ -0,0 +1,31 @@
+# Code Owners
+
+## ALLOWED_UNOWNED_FILES.yml File Format
+
+This file is for repos that require all files be owned. Some files may be listed here as an exception and will be added to the end of the CODEOWNERS.
+
+`version` is the current version of the `ALLOWED_UNOWNED_FILES.yml` file format. The only version is `1.0.0`.
+
+`filters` are a list of filters that each have a `filter` and `justificaiton` field.
+
+`filter` is a file path. This file path must start with a `/` and is relative to the root repo directory. Directories or globs are not supported at the moment to ensure careful selection of files allowed to be unowned. This can be reconsidered if proper usecases appear.
+
+`justification` is the reason why this file should be unowned. A common case is that this is a generated file that has checks in CI to ensure it is in the correct format.
+
+### Example file
+
+```yaml
+version: 1.0.0 # The version of the file you are using.
+filters: # List of all filters
+  - filter: "/.github/CODEOWNERS" # path to the file, this must be a file and not a directory or glob.
+    justification: "Generated by all of the individual owners files in the repo." # Reason this file should be unowned.
+```
+
+### Configuration
+
+This can be configured in any repo with `bazel_rules_mongo` by putting the following lines in your `.bazelrc` file:
+
+```
+common --define codeowners_have_allowed_unowned_files=True
+common --define codeowners_allowed_unowned_files_path=.github/ALLOWED_UNOWNED_FILES.yml
+```

docs/owners/owners_format.md

@@ -8,7 +8,7 @@ This is loosely based on [kubernetes](https://www.kubernetes.dev/docs/guide/owne
 
 `aliases` point to yaml files files that list aliases that can be used in this OWNERS.yml file.
 
-`filters` are a list of globs that match [gitignore syntax](https://git-scm.com/docs/gitignore#_pattern_format). The filter must match at least once file and be unique to the file. Each filter must have a list of `approvers`. An approval from any single approver will allow the code to be merged. Each filter can optionally have a `metadata` tag. Inside that tag a user can put whatever tags they want. We have reserved two meaningful tags `emeritus_approvers` and `owning_team`. This is not an exhaustive list and more documented and undocumented options can be added later. There is no linting done on the metadata tag.
+`filters` are a list of globs that match [gitignore syntax](https://git-scm.com/docs/gitignore#_pattern_format). The filter must match at least once file and be unique to the file. Each filter must have a list of `approvers`. An approval from any single approver will allow the code to be merged. `NOOWNER` can be specified to mark a filter as unowned. Each filter can optionally have a `metadata` tag. Inside that tag a user can put whatever tags they want. We have reserved two meaningful tags `emeritus_approvers` and `owning_team`. This is not an exhaustive list and more documented and undocumented options can be added later. There is no linting done on the metadata tag.
 
 `emeritus_approvers` are folks that used to be approvers that no longer have approver privileges. This allows us to keep track of folks who built up a knowledge base of this code that might need to be consulted in a critical situation. Both `approvers` and `emeritus_approvers` should be either github usernames, emails, or aliases.
 

src/mongo/base/OWNERS.yml

@@ -5,7 +5,7 @@ filters:
       - 10gen/server-programmability
   - "error_codes.yml":
     approvers:
-      - NOOWNERS-DO-NOT-USE-DEPRECATED-2024-07-01
+      - NOOWNERS
   - "secure_allocator*":
     approvers:
       - 10gen/server-security

src/mongo/transport/OWNERS.yml

@@ -5,4 +5,4 @@ filters:
       - 10gen/server-networking-and-observability
   - "ssl_connection_context.h":
     approvers:
-      - NOOWNERS-DO-NOT-USE-DEPRECATED-2024-07-01
+      - NOOWNERS