From 9b01b02ef54511bdf2a944bd917a49896bb64e8d Mon Sep 17 00:00:00 2001 From: Juan Gu <111318390+juangugit@users.noreply.github.com> Date: Thu, 30 Jan 2025 11:33:02 -0800 Subject: [PATCH] SERVER-98403 Lint redundant/bad CODEOWNERS rules (#31416) GitOrigin-RevId: cc41658a1e273216772c363c49b93170d695dbe9 --- .github/BUILD.bazel | 5 + BUILD.bazel | 1 + buildscripts/BUILD.bazel | 42 ++++- buildscripts/codeowners_generate.py | 26 ++- buildscripts/download_codeowners_validator.py | 153 ++++++++++++++++++ buildscripts/validate_codeowners.py | 87 ++++++++++ 6 files changed, 311 insertions(+), 3 deletions(-) create mode 100644 .github/BUILD.bazel create mode 100755 buildscripts/download_codeowners_validator.py create mode 100755 buildscripts/validate_codeowners.py diff --git a/.github/BUILD.bazel b/.github/BUILD.bazel new file mode 100644 index 00000000000..f2a623cf8d5 --- /dev/null +++ b/.github/BUILD.bazel @@ -0,0 +1,5 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "CODEOWNERS", +]) diff --git a/BUILD.bazel b/BUILD.bazel index 87dfe4f35e6..4c27d7dd957 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -11,6 +11,7 @@ exports_files([ "pyproject.toml", "poetry.lock", "symbols.orderfile", + "codeowners-validator", ]) npm_link_all_packages(name = "node_modules") diff --git a/buildscripts/BUILD.bazel b/buildscripts/BUILD.bazel index b3e2d46f105..59872c27e35 100644 --- a/buildscripts/BUILD.bazel +++ b/buildscripts/BUILD.bazel @@ -7,7 +7,11 @@ exports_files([ py_binary( name = "codeowners", - srcs = ["codeowners_generate.py"], + srcs = [ + "codeowners_generate.py", + "download_codeowners_validator.py", + "validate_codeowners.py", + ], main = "codeowners_generate.py", visibility = ["//visibility:public"], deps = [ @@ -15,6 +19,42 @@ py_binary( "pyyaml", group = "core", ), + dependency( + "typer", + group = "core", + ), + ], +) + +py_binary( + name = "download_codeowners_validator", + srcs = ["download_codeowners_validator.py"], + main = "download_codeowners_validator.py", + visibility = ["//visibility:public"], + deps = [ + dependency( + "typer", + group = "core", + ), + ], +) + +py_binary( + name = "validate_codeowners", + srcs = [ + "download_codeowners_validator.py", + "validate_codeowners.py", + ], + data = [ + "//.github:CODEOWNERS", + ], + main = "validate_codeowners.py", + visibility = ["//visibility:public"], + deps = [ + dependency( + "typer", + group = "core", + ), ], ) diff --git a/buildscripts/codeowners_generate.py b/buildscripts/codeowners_generate.py index 51d9f32672d..72617e8d09d 100644 --- a/buildscripts/codeowners_generate.py +++ b/buildscripts/codeowners_generate.py @@ -9,6 +9,8 @@ from functools import lru_cache import yaml +from buildscripts.validate_codeowners import run_validator + OWNERS_FILE_NAME = "OWNERS.yml" @@ -170,6 +172,25 @@ def print_diff_and_instructions(old_codeowners_contents, new_codeowners_contents print("python buildscripts/install_bazel.py") +def validate_generated_codeowners() -> int: + """Validate the generated CODEOWNERS file. + + Returns: + int: 0 if validation succeeds, non-zero otherwise. + """ + print("\nValidating generated CODEOWNERS file...") + try: + validation_result = run_validator("./") + if validation_result != 0: + print("CODEOWNERS validation failed!", file=sys.stderr) + return validation_result + print("CODEOWNERS validation successful!") + return 0 + except Exception as exc: + print(f"Error during CODEOWNERS validation: {str(exc)}", file=sys.stderr) + return 1 + + def main(): # If we are running in bazel, default the directory to the workspace default_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY") @@ -245,13 +266,14 @@ def main(): return 1 print("CODEOWNERS file is up to date") - return 0 + return validate_generated_codeowners() with open(output_file, "w") as file: file.write(new_contents) print(f"Successfully wrote to the CODEOWNERS file at: {os.path.abspath(output_file)}") - return 0 + # Add validation after generating CODEOWNERS file + return validate_generated_codeowners() if __name__ == "__main__": diff --git a/buildscripts/download_codeowners_validator.py b/buildscripts/download_codeowners_validator.py new file mode 100755 index 00000000000..47095bca160 --- /dev/null +++ b/buildscripts/download_codeowners_validator.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Script for downloading codeowners-validator.""" + +import hashlib +import os +import platform +import stat +import tarfile +import urllib.request +import zipfile +from typing import Annotated + +import typer + +VALIDATOR_VERSION = "0.7.4" +VALIDATOR_BINARY_NAME = "codeowners-validator" +RELEASE_URL = ( + f"https://github.com/mszostok/codeowners-validator/releases/download/v{VALIDATOR_VERSION}/" +) + +VALIDATOR_SHA256 = { + "windows": { + "x86_64": "4e71fcc686ad4f275a2fe9af0e3290a443e2907b07bd946f174109d5c057cda5", + "i386": "08bc65d8773b264a1e45d7589d53aab05ce697f1ddb46ba15e0c7468b0574fb6", + }, + "linux": { + "x86_64": "73677228e3c7ddf3f9296246f2c088375c5b1b82385069360867d026ab790028", + "i386": "6384762879d26c886da2a3ae9e99b076d1cf35749173ae99fea1ebb6c245b094", + "arm64": "9286238af043e3bd42b2309530844e14ed51ec6c5c1aac9ac034068eb8d78668", + }, + "darwin": { + "x86_64": "25ae64da52eb2aad357af1275adb46fdf0c2d560def1cb9479800c775e65aa8e", + "arm64": "efa77bf32d971181e007825a9c1b1239a690d8bab56e3e606e010c61511fd19e", + }, +} + + +def determine_platform(): + """Determine the operating system.""" + syst = platform.system() + pltf = None + if syst == "Darwin": + pltf = "darwin" + elif syst == "Windows": + pltf = "windows" + elif syst == "Linux": + pltf = "linux" + else: + raise RuntimeError("Platform cannot be inferred.") + return pltf + + +def determine_architecture(): + """Determine the CPU architecture.""" + arch = None + machine = platform.machine().lower() + if machine in ("amd64", "x86_64"): + arch = "x86_64" + elif machine in ("arm64", "aarch64"): + arch = "arm64" + elif machine in ("i386", "i686", "x86"): + arch = "i386" + else: + raise RuntimeError(f"Detected architecture is not supported: {machine}") + return arch + + +def sha256_file(filename: str) -> str: + sha256_hash = hashlib.sha256() + with open(filename, "rb") as f: + for block in iter(lambda: f.read(4096), b""): + sha256_hash.update(block) + return sha256_hash.hexdigest() + + +def download_validator_binary(download_location: str): + """Download the codeowners-validator binary.""" + + # expand user to get absolute path + download_location = os.path.expanduser(download_location) + workspace_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", ".") + if workspace_dir: + download_location = os.path.join(workspace_dir, download_location) + + operating_system = determine_platform() + architecture = determine_architecture() + + if operating_system == "windows": + extension = ".zip" + else: + extension = ".tar.gz" + + binary_name = ( + f"{VALIDATOR_BINARY_NAME}_{VALIDATOR_VERSION}_{operating_system}_{architecture}{extension}" + ) + url = f"{RELEASE_URL}{binary_name}" + + # Download the archive + archive_location = os.path.join(download_location, binary_name) + urllib.request.urlretrieve(url, archive_location) + print(f"Downloaded codeowners-validator from {url} to {archive_location}") + + # Extract archive + if operating_system == "windows": + with zipfile.ZipFile(archive_location) as zip_ref: + for file_name in zip_ref.namelist(): + if file_name == VALIDATOR_BINARY_NAME: + zip_ref.extract(file_name, download_location) + else: + with tarfile.open(archive_location) as tar: + for member in tar.getmembers(): + if member.name == VALIDATOR_BINARY_NAME: + tar.extract(member, download_location) + + binary_path = os.path.join( + download_location, VALIDATOR_BINARY_NAME + (".exe" if operating_system == "windows" else "") + ) + + expected_sha = VALIDATOR_SHA256.get(operating_system, {}).get(architecture) + print(f"Expected SHA256: {expected_sha}") + if not expected_sha: + raise RuntimeError(f"No SHA256 hash found for {operating_system}/{architecture}") + + calculated_sha = sha256_file(binary_path) + print(f"Calculated SHA256: {calculated_sha}") + if calculated_sha != expected_sha: + raise RuntimeError( + f"Downloaded file from {url} calculated sha ({calculated_sha}) did not match expected sha ({expected_sha})" + ) + + # Set executable permissions on Unix-like systems + if operating_system != "windows": + os.chmod(binary_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + print(f"Set user executable permissions on {binary_path}") + + # Clean up archive + os.remove(archive_location) + + +def main( + download_location: Annotated[ + str, + typer.Option( + help="Directory to download the codeowners-validator binary to.", + ), + ] = "./", +): + """Downloads codeowners-validator for use in evergreen and local development.""" + download_validator_binary(download_location=download_location) + + +if __name__ == "__main__": + typer.run(main) diff --git a/buildscripts/validate_codeowners.py b/buildscripts/validate_codeowners.py new file mode 100755 index 00000000000..fac71af74cd --- /dev/null +++ b/buildscripts/validate_codeowners.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Script for validating CODEOWNERS file.""" + +import os +import subprocess +import sys +from typing import Annotated + +import typer + +from buildscripts.download_codeowners_validator import download_validator_binary + + +def get_validator_env() -> dict: + """Prepare the environment for the codeowners-validator.""" + env = os.environ.copy() + + env.update( + { + "REPOSITORY_PATH": ".", + "CHECKS": "duppatterns,syntax", + "EXPERIMENTAL_CHECKS": "avoid-shadowing", + "OWNER_CHECKER_REPOSITORY": "10gen/mongo", + } + ) + return env + + +def run_validator(validator_path: str) -> int: + """Run the codeowners validation.""" + validator_path = os.path.join(os.path.expanduser(validator_path), "codeowners-validator") + downloaded_by_current_script = False + # If we are running in bazel, default the directory to the workspace + workspace_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", ".") + if workspace_dir: + validator_path = os.path.join(workspace_dir, validator_path) + + if not os.path.isfile(validator_path): + print(f"Validator not found at {validator_path}, attempting to download...") + try: + download_validator_binary(os.path.dirname(validator_path)) + if not os.path.isfile(validator_path): + print( + f"Error: Validator still not found at {validator_path} after download attempt", + file=sys.stderr, + ) + return 1 + downloaded_by_current_script = True + except Exception as exc: + print(f"Failed to download validator: {str(exc)}", file=sys.stderr) + return 1 + + print(f"Using validator at: {validator_path}") + env = get_validator_env() + + try: + result = subprocess.run( + [validator_path], env=env, check=True, capture_output=True, text=True + ) + if result.stdout: + print(result.stdout) + return 0 + except subprocess.CalledProcessError as exc: + if exc.stdout: + print(exc.stdout, file=sys.stderr) + if exc.stderr: + print(exc.stderr, file=sys.stderr) + return exc.returncode + except FileNotFoundError: + print("Error: Failed to run codeowners-validator after installation", file=sys.stderr) + return 1 + finally: + if downloaded_by_current_script and os.path.isfile(validator_path): + os.remove(validator_path) + + +def main( + validator_path: Annotated[ + str, typer.Option(help="Path to the codeowners-validator binary") + ] = "./", +) -> int: + """Validate CODEOWNERS file using codeowners-validator.""" + return run_validator(validator_path=validator_path) + + +if __name__ == "__main__": + typer.run(main)